ASAN errors in hp_key_cmp / Expression_cache_tmptable::check_value upon mix of collations and data types

CREATE TABLE t1 (a INT);

INSERT INTO t1 VALUES (1),(2),(3);

CREATE TABLE t2 (b BIT(1), c VARCHAR(8) NOT NULL) ENGINE=MyISAM CHARACTER SET cp1250 COLLATE cp1250_czech_cs;

INSERT INTO t2 VALUES (b'0','foo'),(b'1','bar');

CREATE TABLE t3 (d VARCHAR(8)) ENGINE=MyISAM CHARACTER SET cp1250 COLLATE cp1250_czech_cs;

SELECT t1.* FROM (t1, t2) WHERE b IN (SELECT d FROM t3 WHERE d = c) OR a = 43;

# Cleanup

DROP TABLE t1, t2, t3;

==3834594==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x6290002577c4 at pc 0x55d04103a13b bp 0x7f856507e670 sp 0x7f856507e668

READ of size 1 at 0x6290002577c4 thread T5

    #0 0x55d04103a13a in my_strnncollsp_win1250ch /data/bld/10.5-asan/strings/ctype-win1250ch.c:485

    #1 0x55d040358c3e in my_ci_strnncollsp /data/bld/10.5-asan/include/m_ctype.h:1139

    #2 0x55d04035d142 in hp_key_cmp /data/bld/10.5-asan/storage/heap/hp_hash.c:550

    #3 0x55d0403597ff in hp_search /data/bld/10.5-asan/storage/heap/hp_hash.c:130

    #4 0x55d0403642df in heap_rkey /data/bld/10.5-asan/storage/heap/hp_rkey.c:63

    #5 0x55d040351070 in ha_heap::index_read_map(unsigned char*, unsigned char const*, unsigned long, ha_rkey_function) /data/bld/10.5-asan/storage/heap/ha_heap.cc:292

    #6 0x55d03f92a3c2 in handler::ha_index_read_map(unsigned char*, unsigned char const*, unsigned long, ha_rkey_function) /data/bld/10.5-asan/sql/handler.cc:3244

    #7 0x55d03f21ca9b in join_read_key2(THD*, st_join_table*, TABLE*, st_table_ref*) /data/bld/10.5-asan/sql/sql_select.cc:21987

    #8 0x55d03f63a899 in Expression_cache_tmptable::check_value(Item**) /data/bld/10.5-asan/sql/sql_expression_cache.cc:223

    #9 0x55d03f9b200e in Item_cache_wrapper::check_cache() /data/bld/10.5-asan/sql/item.cc:8991

    #10 0x55d03f9b488d in Item_cache_wrapper::val_bool() /data/bld/10.5-asan/sql/item.cc:9186

    #11 0x55d03fa14d64 in Item_cond_or::val_int() /data/bld/10.5-asan/sql/item_cmpfunc.cc:5585

    #12 0x55d03f3f5988 in SQL_SELECT::skip_record(THD*) /data/bld/10.5-asan/sql/opt_range.h:1726

    #13 0x55d03f5df1a4 in JOIN_CACHE::check_match(unsigned char*) /data/bld/10.5-asan/sql/sql_join_cache.cc:2584

    #14 0x55d03f5d1f75 in JOIN_CACHE::generate_full_extensions(unsigned char*) /data/bld/10.5-asan/sql/sql_join_cache.cc:2527

    #15 0x55d03f5d193a in JOIN_CACHE::join_matching_records(bool) /data/bld/10.5-asan/sql/sql_join_cache.cc:2427

    #16 0x55d03f5cfc3d in JOIN_CACHE::join_records(bool) /data/bld/10.5-asan/sql/sql_join_cache.cc:2178

    #17 0x55d03f216c2a in sub_select_cache(JOIN*, st_join_table*, bool) /data/bld/10.5-asan/sql/sql_select.cc:21065

    #18 0x55d03f21738b in sub_select(JOIN*, st_join_table*, bool) /data/bld/10.5-asan/sql/sql_select.cc:21247

    #19 0x55d03f215ae9 in do_select /data/bld/10.5-asan/sql/sql_select.cc:20825

    #20 0x55d03f1a0b34 in JOIN::exec_inner() /data/bld/10.5-asan/sql/sql_select.cc:4661

    #21 0x55d03f19e11d in JOIN::exec() /data/bld/10.5-asan/sql/sql_select.cc:4441

    #22 0x55d03f1a2434 in mysql_select(THD*, TABLE_LIST*, List<Item>&, Item*, unsigned int, st_order*, st_order*, Item*, st_order*, unsigned long long, select_result*, st_select_lex_unit*, st_select_lex*) /data/bld/10.5-asan/sql/sql_select.cc:4918

    #23 0x55d03f17262b in handle_select(THD*, LEX*, select_result*, unsigned long) /data/bld/10.5-asan/sql/sql_select.cc:449

    #24 0x55d03f0d922c in execute_sqlcom_select /data/bld/10.5-asan/sql/sql_parse.cc:6437

    #25 0x55d03f0c766e in mysql_execute_command(THD*) /data/bld/10.5-asan/sql/sql_parse.cc:4029

    #26 0x55d03f0e403a in mysql_parse(THD*, char*, unsigned int, Parser_state*, bool, bool) /data/bld/10.5-asan/sql/sql_parse.cc:8237

    #27 0x55d03f0b9320 in dispatch_command(enum_server_command, THD*, char*, unsigned int, bool, bool) /data/bld/10.5-asan/sql/sql_parse.cc:1891

    #28 0x55d03f0b5cb3 in do_command(THD*) /data/bld/10.5-asan/sql/sql_parse.cc:1375

    #29 0x55d03f50f868 in do_handle_one_connection(CONNECT*, bool) /data/bld/10.5-asan/sql/sql_connect.cc:1407

    #30 0x55d03f50f3ce in handle_one_connection /data/bld/10.5-asan/sql/sql_connect.cc:1319

    #31 0x55d04016d1ef in pfs_spawn_thread /data/bld/10.5-asan/storage/perfschema/pfs.cc:2201

    #32 0x7f856e8a8043 in start_thread nptl/pthread_create.c:442

    #33 0x7f856e92861b in clone3 ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

0x6290002577c4 is located 1452 bytes to the right of 16408-byte region [0x629000253200,0x629000257218)

allocated by thread T5 here:

    #0 0x7f856f2b89cf in __interceptor_malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69

    #1 0x55d040f1f873 in my_malloc /data/bld/10.5-asan/mysys/my_malloc.c:91

    #2 0x55d040ec342a in init_io_cache_ext /data/bld/10.5-asan/mysys/mf_iocache.c:248

    #3 0x55d040ec3c55 in init_io_cache /data/bld/10.5-asan/mysys/mf_iocache.c:301

    #4 0x55d040df7954 in mi_extra /data/bld/10.5-asan/storage/myisam/mi_extra.c:99

    #5 0x55d040d9cd3c in ha_myisam::extra_opt(ha_extra_function, unsigned long) /data/bld/10.5-asan/storage/myisam/ha_myisam.cc:2189

    #6 0x55d03fd8b261 in init_read_record(READ_RECORD*, THD*, TABLE*, SQL_SELECT*, SORT_INFO*, int, bool, bool) /data/bld/10.5-asan/sql/records.cc:337

    #7 0x55d03f21e8eb in join_init_read_record(st_join_table*) /data/bld/10.5-asan/sql/sql_select.cc:22268

    #8 0x55d03f217aa6 in sub_select(JOIN*, st_join_table*, bool) /data/bld/10.5-asan/sql/sql_select.cc:21303

    #9 0x55d03f2159e3 in do_select /data/bld/10.5-asan/sql/sql_select.cc:20823

    #10 0x55d03f1a0b34 in JOIN::exec_inner() /data/bld/10.5-asan/sql/sql_select.cc:4661

    #11 0x55d03f19e11d in JOIN::exec() /data/bld/10.5-asan/sql/sql_select.cc:4441

    #12 0x55d03f1a2434 in mysql_select(THD*, TABLE_LIST*, List<Item>&, Item*, unsigned int, st_order*, st_order*, Item*, st_order*, unsigned long long, select_result*, st_select_lex_unit*, st_select_lex*) /data/bld/10.5-asan/sql/sql_select.cc:4918

    #13 0x55d03f17262b in handle_select(THD*, LEX*, select_result*, unsigned long) /data/bld/10.5-asan/sql/sql_select.cc:449

    #14 0x55d03f0d922c in execute_sqlcom_select /data/bld/10.5-asan/sql/sql_parse.cc:6437

    #15 0x55d03f0c766e in mysql_execute_command(THD*) /data/bld/10.5-asan/sql/sql_parse.cc:4029

    #16 0x55d03f0e403a in mysql_parse(THD*, char*, unsigned int, Parser_state*, bool, bool) /data/bld/10.5-asan/sql/sql_parse.cc:8237

    #17 0x55d03f0b9320 in dispatch_command(enum_server_command, THD*, char*, unsigned int, bool, bool) /data/bld/10.5-asan/sql/sql_parse.cc:1891

    #18 0x55d03f0b5cb3 in do_command(THD*) /data/bld/10.5-asan/sql/sql_parse.cc:1375

    #19 0x55d03f50f868 in do_handle_one_connection(CONNECT*, bool) /data/bld/10.5-asan/sql/sql_connect.cc:1407

    #20 0x55d03f50f3ce in handle_one_connection /data/bld/10.5-asan/sql/sql_connect.cc:1319

    #21 0x55d04016d1ef in pfs_spawn_thread /data/bld/10.5-asan/storage/perfschema/pfs.cc:2201

    #22 0x7f856e8a8043 in start_thread nptl/pthread_create.c:442

Thread T5 created by T0 here:

    #0 0x7f856f249726 in __interceptor_pthread_create ../../../../src/libsanitizer/asan/asan_interceptors.cpp:207

    #1 0x55d040168f22 in my_thread_create /data/bld/10.5-asan/storage/perfschema/my_thread.h:52

    #2 0x55d04016d5de in pfs_spawn_thread_v1 /data/bld/10.5-asan/storage/perfschema/pfs.cc:2252

    #3 0x55d03ed90faa in inline_mysql_thread_create /data/bld/10.5-asan/include/mysql/psi/mysql_thread.h:1323

    #4 0x55d03eda71c7 in create_thread_to_handle_connection(CONNECT*) /data/bld/10.5-asan/sql/mysqld.cc:6116

    #5 0x55d03eda77d8 in create_new_thread(CONNECT*) /data/bld/10.5-asan/sql/mysqld.cc:6175

    #6 0x55d03eda7aab in handle_accepted_socket(st_mysql_socket, st_mysql_socket) /data/bld/10.5-asan/sql/mysqld.cc:6240

    #7 0x55d03eda869c in handle_connections_sockets() /data/bld/10.5-asan/sql/mysqld.cc:6367

    #8 0x55d03eda5531 in run_main_loop /data/bld/10.5-asan/sql/mysqld.cc:5357

    #9 0x55d03eda6a96 in mysqld_main(int, char**) /data/bld/10.5-asan/sql/mysqld.cc:5768

    #10 0x55d03ed8f918 in main /data/bld/10.5-asan/sql/main.cc:25

    #11 0x7f856e8461c9 in __libc_start_call_main ../sysdeps/nptl/libc_start_call_main.h:58

SUMMARY: AddressSanitizer: heap-buffer-overflow /data/bld/10.5-asan/strings/ctype-win1250ch.c:485 in my_strnncollsp_win1250ch

Shadow bytes around the buggy address:

  0x0c5280042ea0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa

  0x0c5280042eb0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa

  0x0c5280042ec0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa

  0x0c5280042ed0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa

  0x0c5280042ee0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa

=>0x0c5280042ef0: fa fa fa fa fa fa fa fa[fa]fa fa fa fa fa fa fa

  0x0c5280042f00: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa

  0x0c5280042f10: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa

  0x0c5280042f20: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa

  0x0c5280042f30: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa

  0x0c5280042f40: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa

Shadow byte legend (one shadow byte represents 8 application bytes):

  Addressable:           00

  Partially addressable: 01 02 03 04 05 06 07 

  Heap left redzone:       fa

  Freed heap region:       fd

  Stack left redzone:      f1

  Stack mid redzone:       f2

  Stack right redzone:     f3

  Stack after return:      f5

  Stack use after scope:   f8

  Global redzone:          f9

  Global init order:       f6

  Poisoned by user:        f7

  Container overflow:      fc

  Array cookie:            ac

  Intra object redzone:    bb

  ASan internal:           fe

  Left alloca redzone:     ca

  Right alloca redzone:    cb

==3834594==ABORTING