Uploaded image for project: 'MariaDB Server'
  1. MariaDB Server
  2. MDEV-12465

Server crashes in my_scan_weight_utf8_bin upon collecting stats for RocksDB table

Details

    Description

      10.2 ebce682557

      		 
      #3  <signal handler called>
      		 
      #4  0x00007fe6661f6249 in my_scan_weight_utf8_bin (weight=0x7fe65d5f88dc, str=0xa5a5a5a5a5a5a5a5 <error: Cannot access memory at address 0xa5a5a5a5a5a5a5a5>, end=0xa5a5a5a5a5a5a8a2 <error: Cannot access memory at address 0xa5a5a5a5a5a5a8a2>) at /data/src/10.2/strings/strcoll.ic:89
      		 
      #5  0x00007fe6661f6525 in my_strnncollsp_utf8_bin (cs=0x7fe666d0da40 <my_charset_utf8_bin>, a=0xa5a5a5a5a5a5a5a5 <error: Cannot access memory at address 0xa5a5a5a5a5a5a5a5>, a_length=765, b=0xa5a5a5a5a5a5a5a5 <error: Cannot access memory at address 0xa5a5a5a5a5a5a5a5>, b_length=765) at /data/src/10.2/strings/strcoll.ic:245
      		 
      #6  0x00007fe665a2e781 in Field_blob::cmp (this=0x7fe64c289918, a=0xa5a5a5a5a5a5a5a5 <error: Cannot access memory at address 0xa5a5a5a5a5a5a5a5>, a_length=765, b=0xa5a5a5a5a5a5a5a5 <error: Cannot access memory at address 0xa5a5a5a5a5a5a5a5>, b_length=765) at /data/src/10.2/sql/field.cc:8097
      		 
      #7  0x00007fe665a2e834 in Field_blob::cmp_max (this=0x7fe64c289918, a_ptr=0x7fe64c251904 '\245' <repeats 200 times>..., b_ptr=0x7fe64c2512ed "\245\245\245\245\245\245\245\245\245\245\001", max_length=765) at /data/src/10.2/sql/field.cc:8110
      		 
      #8  0x00007fe665b3fad5 in key_rec_cmp (key_p=0x7fe64c2204c8, first_rec=0x7fe64c2512f9 "\245\002", second_rec=0x7fe64c250ce2 "\245\001") at /data/src/10.2/sql/key.cc:612
      		 
      #9  0x00007fe666175284 in cmp_key_part_id (key_p=0x7fe64c2204c8, ref1=0x7fe64c2512f7 "\001", ref2=0x7fe64c250ce0 "") at /data/src/10.2/sql/ha_partition.cc:5390
      		 
      #10 0x00007fe6661ac6a5 in _downheap (queue=0x7fe64c2204f0, start_idx=1, element=0x7fe64c250ce0 "") at /data/src/10.2/mysys/queues.c:305
      		 
      #11 0x00007fe6661ac844 in queue_fix (queue=0x7fe64c2204f0) at /data/src/10.2/mysys/queues.c:354
      		 
      #12 0x00007fe666176d57 in ha_partition::handle_ordered_index_scan (this=0x7fe64c21fff8, buf=0x7fe64c288b68 "\372\002", reverse_order=false) at /data/src/10.2/sql/ha_partition.cc:6200
      		 
      #13 0x00007fe666175807 in ha_partition::common_first_last (this=0x7fe64c21fff8, buf=0x7fe64c288b68 "\372\002") at /data/src/10.2/sql/ha_partition.cc:5588
      		 
      #14 0x00007fe6661756ae in ha_partition::index_first (this=0x7fe64c21fff8, buf=0x7fe64c288b68 "\372\002") at /data/src/10.2/sql/ha_partition.cc:5536
      		 
      #15 0x00007fe665a4e931 in handler::ha_index_first (this=0x7fe64c21fff8, buf=0x7fe64c288b68 "\372\002") at /data/src/10.2/sql/handler.cc:2725
      		 
      #16 0x00007fe66588c4c7 in collect_statistics_for_index (thd=0x7fe64c000b00, table=0x7fe64c286be0, index=1) at /data/src/10.2/sql/sql_statistics.cc:2638
      		 
      #17 0x00007fe66588ca58 in collect_statistics_for_table (thd=0x7fe64c000b00, table=0x7fe64c286be0) at /data/src/10.2/sql/sql_statistics.cc:2807
      		 
      #18 0x00007fe66593434b in mysql_admin_table(THD *, TABLE_LIST *, HA_CHECK_OPT *, const char *, thr_lock_type, bool, bool, uint, int (*)(THD *, TABLE_LIST *, HA_CHECK_OPT *), struct {...}, int (*)(THD *, TABLE_LIST *, HA_CHECK_OPT *)) (thd=0x7fe64c000b00, tables=0x7fe64c012440, check_opt=0x7fe64c0054a0, operator_name=0x7fe6662736b7 "analyze", lock_type=TL_READ_NO_INSERT, open_for_modify=true, repair_table_use_frm=false, extra_open_options=0, prepare_func=0x0, operator_func=(int (handler::*)(handler * const, THD *, HA_CHECK_OPT *)) 0x7fe665a518ba <handler::ha_analyze(THD*, st_ha_check_opt*)>, view_operator_func=0x0) at /data/src/10.2/sql/sql_admin.cc:877
      		 
      #19 0x00007fe665935d59 in Sql_cmd_analyze_table::execute (this=0x7fe64c012a50, thd=0x7fe64c000b00) at /data/src/10.2/sql/sql_admin.cc:1306
      		 
      #20 0x00007fe6657e6119 in mysql_execute_command (thd=0x7fe64c000b00) at /data/src/10.2/sql/sql_parse.cc:6200
      		 
      #21 0x00007fe6657ea9da in mysql_parse (thd=0x7fe64c000b00, rawbuf=0x7fe64c012348 "ANALYZE TABLE t1 PERSISTENT FOR ALL", length=35, parser_state=0x7fe65d5fa210, is_com_multi=false, is_next_command=false) at /data/src/10.2/sql/sql_parse.cc:7874
      		 
      #22 0x00007fe6657d8ab2 in dispatch_command (command=COM_QUERY, thd=0x7fe64c000b00, packet=0x7fe64c0dfa01 "", packet_length=35, is_com_multi=false, is_next_command=false) at /data/src/10.2/sql/sql_parse.cc:1812
      		 
      #23 0x00007fe6657d7422 in do_command (thd=0x7fe64c000b00) at /data/src/10.2/sql/sql_parse.cc:1362
      		 
      #24 0x00007fe6659219e9 in do_handle_one_connection (connect=0x7fe667a5e700) at /data/src/10.2/sql/sql_connect.cc:1354
      		 
      #25 0x00007fe665921776 in handle_one_connection (arg=0x7fe667a5e700) at /data/src/10.2/sql/sql_connect.cc:1260
      		 
      #26 0x00007fe665d3a9d8 in pfs_spawn_thread (arg=0x7fe6679bc170) at /data/src/10.2/storage/perfschema/pfs.cc:1862
      		 
      #27 0x00007fe664e00064 in start_thread (arg=0x7fe65d5fb700) at pthread_create.c:309
      		 
      #28 0x00007fe6631f062d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111


      --source include/have_partition.inc
      		 
      INSTALL SONAME 'ha_rocksdb';
      		 
       
      		 
      CREATE TABLE t1 (
      		 
        pk INT,
      		 
        f1 CHAR(255),
      		 
        f2 TEXT,
      		 
        f3 VARCHAR(255),
      		 
        f4 TEXT,
      		 
        PRIMARY KEY (pk),
      		 
        KEY (f4(255))
      		 
      ) ENGINE=RocksDB
      		 
        CHARSET utf8
      		 
        COLLATE utf8_bin
      		 
        PARTITION BY KEY (pk) PARTITIONS 2;
      		 
      INSERT INTO t1 VALUES
      		 
      (1,'foo','bar','foo','bar'), (2,'bar','foo','bar','foo');
      		 
       
      		 
      ANALYZE TABLE t1 PERSISTENT FOR ALL;

      Attachments

        Activity

          Ascending order - Click to sort in descending order
          psergei Sergei Petrunia added a comment -

          Analysis.

          The crash happens when EITS code is doing an ordered index scan for this index:

            f4 TEXT,
          		 
          ...
          		 
            KEY (f4(255))

          ha_partition handles ordered index scans that read data from multiple partitions by running ordering index scans on all the partitions and using a priority queue to merge multiple ordered streams of rows into one.

          In order to do the merge, it needs to compare the index tuples.

          On the other hand, EITS code wants the index scan to be fast, in order to do that, collect_statistics_for_index makes this call:

            table->file->ha_start_keyread(index);

          So,

          • ha_partition passes it to its partitions, MyRocks honors this call and does an index-only scan.
          • Then, it doesn't read anything to the place for column 'f4'.
          • Then, ha_partition tries to compare index columns for two partitions.
          • Both have uninitialized data, and we get a crash.
          psergei Sergei Petrunia added a comment - Analysis. The crash happens when EITS code is doing an ordered index scan for this index: f4 TEXT, ... KEY (f4(255)) ha_partition handles ordered index scans that read data from multiple partitions by running ordering index scans on all the partitions and using a priority queue to merge multiple ordered streams of rows into one. In order to do the merge, it needs to compare the index tuples. On the other hand, EITS code wants the index scan to be fast, in order to do that, collect_statistics_for_index makes this call: table->file->ha_start_keyread(index); So, ha_partition passes it to its partitions, MyRocks honors this call and does an index-only scan. Then, it doesn't read anything to the place for column 'f4'. Then, ha_partition tries to compare index columns for two partitions. Both have uninitialized data, and we get a crash.
          psergei Sergei Petrunia added a comment -

          One may wonder why this problem only affects MyRocks and not InnoDB.

          I've debugged it:

          • InnoDB also gets extra(HA_EXTRA_KEYREAD) call and notes it should do an index-only scan
          • But then, ha_innobase::change_active_index calls ha_innobase::build_template, build_template_needs_field which have the logic to turn "index-only" OFF whenever there is a column read that "is at least partially covered by the used index and is present in the read_set" (or something like that). Stack trace of the point where index-only gets disabled: https://gist.github.com/spetrunia/cbb0d36c4ccd2d5172192ffb3f8da7b7
          psergei Sergei Petrunia added a comment - One may wonder why this problem only affects MyRocks and not InnoDB. I've debugged it: InnoDB also gets extra(HA_EXTRA_KEYREAD) call and notes it should do an index-only scan But then, ha_innobase::change_active_index calls ha_innobase::build_template, build_template_needs_field which have the logic to turn "index-only" OFF whenever there is a column read that "is at least partially covered by the used index and is present in the read_set" (or something like that). Stack trace of the point where index-only gets disabled: https://gist.github.com/spetrunia/cbb0d36c4ccd2d5172192ffb3f8da7b7
          psergei Sergei Petrunia added a comment -

          Not sure which way should this be fixed :

          • Make MyRocks behave like InnoDB does
          • Make EITS not request index-only reads when they are not really doable anyway?

          Will discuss on the optimizer call.

          psergei Sergei Petrunia added a comment - Not sure which way should this be fixed : Make MyRocks behave like InnoDB does Make EITS not request index-only reads when they are not really doable anyway? Will discuss on the optimizer call.
          alice Alice Sherepa added a comment -

          in MDEV-14391 Innodb crash, stacktrace looks very alike

          alice Alice Sherepa added a comment - in MDEV-14391 Innodb crash, stacktrace looks very alike
          marko Marko Mäkelä added a comment -

          I believe that MDEV-14391 is unrelated to this. It is about indexed virtual columns, while this one appears to be about engine-independent statistics.

          marko Marko Mäkelä added a comment - I believe that MDEV-14391 is unrelated to this. It is about indexed virtual columns, while this one appears to be about engine-independent statistics.
          sanja Oleksandr Byelkin added a comment -

          revision-id: a4107da32bea2c2e442e0bcfe46e1cd9d1e42287 (mariadb-10.2.14-116-ga4107da32be)
          parent(s): 50275321c3ba381d76532067289b3e07e1a3fe26
          author: Oleksandr Byelkin
          committer: Oleksandr Byelkin
          timestamp: 2018-05-17 19:17:28 +0200
          message:

          MDEV-12465: Server crashes in my_scan_weight_utf8_bin upon collecting stats for RocksDB table

          Do not use "only index read" in analyzing indices if there is a field which present in the index only partially.

          —

          sanja Oleksandr Byelkin added a comment - revision-id: a4107da32bea2c2e442e0bcfe46e1cd9d1e42287 (mariadb-10.2.14-116-ga4107da32be) parent(s): 50275321c3ba381d76532067289b3e07e1a3fe26 author: Oleksandr Byelkin committer: Oleksandr Byelkin timestamp: 2018-05-17 19:17:28 +0200 message: MDEV-12465 : Server crashes in my_scan_weight_utf8_bin upon collecting stats for RocksDB table Do not use "only index read" in analyzing indices if there is a field which present in the index only partially. —

          People

            sanja Oleksandr Byelkin
            elenst Elena Stepanova
            Votes:
            0 Vote for this issue
            Watchers:
            5 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved:

              Git Integration

                Error rendering 'com.xiplink.jira.git.jira_git_plugin:git-issue-webpanel'. Please contact your Jira administrators.