Uploaded image for project: 'MariaDB Server'
  1. MariaDB Server
  2. MDEV-23231

[ERROR] [FATAL] InnoDB: Apparent corruption of an index page ... during crash recovery

    XMLWordPrintable

    Details

      Description

      Work flow:
      1. Start server
      2. 33 sessions run concurrend DDL and DML
      3. At some point of time SIGKILL gets sent to the server during 2. is ongoing
      4. Attempt to restart the server
       
      # 2020-07-17T19:57:26 [99933] | [rr 15697 73975]2020-07-17 19:56:49 0 [Note] Recovering after a crash using mysql-bin
      # 2020-07-17T19:57:26 [99933] | [rr 15697 82508]2020-07-17 19:56:53 0 [ERROR] InnoDB: n recs wrong 2817 2816
      # 2020-07-17T19:57:26 [99933] | [rr 15697 82512]2020-07-17 19:56:53 0 [Note] InnoDB: Uncompressed page, stored checksum in field1 673628625, calculated checksums for field1: crc32 673628625, innodb 1039194117,  page type 17855 == INDEX.none 3735928559, stored checksum in field2 673628625, calculated checksums for field2: crc32 673628625, innodb 500201189, none 3735928559,  page LSN 0 33227406, low 4 bytes of LSN at page end 33227406, page number (if stored to page already) 42, space id (if created with >= MySQL-4.1.1 and stored already) 12
      # 2020-07-17T19:57:26 [99933] | [rr 15697 82516]2020-07-17 19:56:53 0 [Note] InnoDB: Page may be an index page where index id is 40
      # 2020-07-17T19:57:26 [99933] | [rr 15697 82520]2020-07-17 19:56:53 0 [Note] InnoDB: Index 40 is `k` in table `test`.`t8`
      # 2020-07-17T19:57:26 [99933] | [rr 15697 82524]2020-07-17 19:56:53 0 [ERROR] [FATAL] InnoDB: Apparent corruption of an index page [page id: space=12, page number=42] to be written to data file. We intentionally crash the server to prevent corrupt data from ending up in data files.
      # 2020-07-17T19:57:26 [99933] | [rr 15697 82543]200717 19:56:53 [rr 15697 82546][ERROR] mysqld got signal 6 ;
       
      (rr) bt
      #0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
      #1  0x0000640d09c208b1 in __GI_abort () at abort.c:79
      #2  0x000055754a97a0ae in ib::fatal::~fatal (this=<optimized out>, __in_chrg=<optimized out>) at /home/mleich/bb-10.4-thiru/storage/innobase/ut/ut0ut.cc:601
      #3  0x000055754aa15ecf in buf_dblwr_assert_on_corrupt_block (block=block@entry=0x757b0cbf4a60) at /home/mleich/bb-10.4-thiru/storage/innobase/buf/buf0dblwr.cc:843
      #4  0x000055754aa16008 in buf_dblwr_check_block (block=block@entry=0x757b0cbf4a60) at /home/mleich/bb-10.4-thiru/storage/innobase/buf/buf0dblwr.cc:898
      #5  0x000055754aa1751d in buf_dblwr_flush_buffered_writes () at /home/mleich/bb-10.4-thiru/storage/innobase/buf/buf0dblwr.cc:1036
      #6  0x000055754aa18654 in buf_dblwr_add_to_batch (bpage=bpage@entry=0x757b0cc11b30) at /home/mleich/bb-10.4-thiru/storage/innobase/buf/buf0dblwr.cc:1170
      #7  0x000055754aa27cac in buf_flush_write_block_low (bpage=bpage@entry=0x757b0cc11b30, flush_type=flush_type@entry=BUF_FLUSH_LRU, sync=sync@entry=false) at /home/mleich/bb-10.4-thiru/storage/innobase/buf/buf0flu.cc:1049
      #8  0x000055754aa2aace in buf_flush_page (buf_pool=buf_pool@entry=0x55754d4415d0, bpage=bpage@entry=0x757b0cc11b30, flush_type=flush_type@entry=BUF_FLUSH_LRU, sync=sync@entry=false) at /home/mleich/bb-10.4-thiru/storage/innobase/buf/buf0flu.cc:1197
      #9  0x000055754aa2c877 in buf_flush_try_neighbors (page_id=page_id@entry=..., flush_type=flush_type@entry=BUF_FLUSH_LRU, n_flushed=119, n_to_flush=n_to_flush@entry=548) at /home/mleich/bb-10.4-thiru/storage/innobase/buf/buf0flu.cc:1423
      #10 0x000055754aa2cb2b in buf_flush_page_and_try_neighbors (bpage=bpage@entry=0x757b0cc0b570, flush_type=flush_type@entry=BUF_FLUSH_LRU, n_to_flush=n_to_flush@entry=548, count=count@entry=0x3dff6cc42c60)
          at /home/mleich/bb-10.4-thiru/storage/innobase/buf/buf0flu.cc:1497
      #11 0x000055754aa2d30d in buf_flush_LRU_list_batch (buf_pool=buf_pool@entry=0x55754d4415d0, max=548, n=n@entry=0x3dff6cc42c60) at /home/mleich/bb-10.4-thiru/storage/innobase/buf/buf0flu.cc:1631
      #12 0x000055754aa2d93a in buf_do_LRU_batch (buf_pool=buf_pool@entry=0x55754d4415d0, max=max@entry=548, n=n@entry=0x3dff6cc42c60) at /home/mleich/bb-10.4-thiru/storage/innobase/buf/buf0flu.cc:1694
      #13 0x000055754aa2db52 in buf_flush_batch (buf_pool=buf_pool@entry=0x55754d4415d0, flush_type=flush_type@entry=BUF_FLUSH_LRU, min_n=min_n@entry=548, lsn_limit=lsn_limit@entry=0, n=n@entry=0x3dff6cc42c60)
          at /home/mleich/bb-10.4-thiru/storage/innobase/buf/buf0flu.cc:1823
      #14 0x000055754aa2dd44 in buf_flush_do_batch (buf_pool=buf_pool@entry=0x55754d4415d0, type=type@entry=BUF_FLUSH_LRU, min_n=548, lsn_limit=lsn_limit@entry=0, n=n@entry=0x3dff6cc42c60) at /home/mleich/bb-10.4-thiru/storage/innobase/buf/buf0flu.cc:1984
      #15 0x000055754aa2e12b in buf_flush_LRU_list (buf_pool=buf_pool@entry=0x55754d4415d0) at /home/mleich/bb-10.4-thiru/storage/innobase/buf/buf0flu.cc:2246
      #16 0x000055754aa2e2a4 in pc_flush_slot () at /home/mleich/bb-10.4-thiru/storage/innobase/buf/buf0flu.cc:2767
      #17 0x000055754aa2f9e3 in buf_flush_page_cleaner_coordinator () at /home/mleich/bb-10.4-thiru/storage/innobase/buf/buf0flu.cc:3196
      #18 0x00006a93691756db in start_thread (arg=0x3dff6cc43700) at pthread_create.c:463
      #19 0x0000640d09d01a3f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
      (rr)
       
      Observed on
      - origin/bb-10.4-thiru a13662dac1df224c607983815b2fe85ee72749c2 2020-07-17T22:49:03+05:30
      - some actual 10.5.5 ~ 2020-07-13
       
      RQG
      git clone https://github.com/mleich1/rqg --branch experimental RQG
      origin/experimental f3d72ce51b150ee1f52a93d18f2024af293d925e 2020-07-13T20:46:53+02:00
       
      perl rqg.pl \                        
      --grammar=conf/mariadb/innodb_compression_encryption.yy \
      --gendata=conf/mariadb/innodb_compression_encryption.zz \
      --mysqld=--plugin-load-add=file_key_management.so \
      --mysqld=--loose-file-key-management-filename=/home/mleich/RQG_mleich/conf/mariadb/encryption_keys.txt \
      --mysqld=--innodb-encrypt-log \
      --mysqld=--innodb-encrypt-tables \
      --mysqld=--innodb_use_native_aio=0 \
      --mysqld=--innodb_lock_schedule_algorithm=fcfs \
      --mysqld=--loose-idle_write_transaction_timeout=0 \
      --mysqld=--loose-idle_transaction_timeout=0 \
      --mysqld=--loose-idle_readonly_transaction_timeout=0 \
      --mysqld=--connect_timeout=60 \
      --mysqld=--interactive_timeout=28800 \
      --mysqld=--slave_net_timeout=60 \
      --mysqld=--net_read_timeout=30 \
      --mysqld=--net_write_timeout=60 \
      --mysqld=--loose-table_lock_wait_timeout=50 \
      --mysqld=--wait_timeout=28800 \
      --mysqld=--lock-wait-timeout=86400 \
      --mysqld=--innodb-lock-wait-timeout=50 \
      --no-mask \
      --queries=10000000 \
      --seed=random \
      --reporters=Backtrace \
      --reporters=ErrorLog \
      --reporters=Deadlock1 \
      --reporters=CrashRecovery1 \
      --validators=None \
      --mysqld=--log_output=none \
      --mysqld=--log-bin \
      --mysqld=--log_bin_trust_function_creators=1 \
      --mysqld=--loose-max-statement-time=30 \
      --mysqld=--loose-debug_assert_on_not_freed_memory=0 \
      --engine=InnoDB \
      --restart_timeout=120 \
      --duration=300 \
      --mysqld=--loose-innodb_fatal_semaphore_wait_threshold=300 \
      --threads=33 \
      --mysqld=--innodb_page_size=64K \
      --mysqld=--innodb-buffer-pool-size=256M \
      --duration=300 \
      --no_mask \
      --workdir=<local settings> \
      --vardir=<local settings> \
      --mtr-build-thread=<local settings> \
      --basedir1=<local settings> \
      --basedir2=<local settings> \
      --script_debug=_nix_ \
      --rr=Server \
      --rr_options=--chaos
      
      

        Attachments

          Issue Links

            Activity

              People

              Assignee:
              thiru Thirunarayanan Balathandayuthapani
              Reporter:
              mleich Matthias Leich
              Votes:
              0 Vote for this issue
              Watchers:
              3 Start watching this issue

                Dates

                Created:
                Updated:
                Resolved:

                  Git Integration