Server crash or ASAN negative-size-param in my_strnncollsp_binary / SORT_FIELD_ATTR::compare_packed_varstrings

CREATE TABLE t1 (a CHAR(240), b BIT(48));

INSERT INTO t1 VALUES ('a',b'0001'),('b',b'0010'),('c',b'0011'),('d',b'0100'),('e',b'0001'),('f',b'0101'),('g',b'0110'),('h',b'0111'),('i',b'1000'),('j',b'1001');

SELECT DES_DECRYPT(a, 'x'), BINARY b FROM t1 GROUP BY 1, 2 WITH ROLLUP;

# Cleanup

DROP TABLE t1;

#3  <signal handler called>

#4  __memcmp_sse4_1 () at ../sysdeps/x86_64/multiarch/memcmp-sse4.S:689

#5  0x000055e5c675010b in my_strnncoll_binary (cs=0x55e5c744cda0 <my_charset_bin>, s=0x7f7a2c063659 "", slen=18446744073709551615, t=0x7f7a2c063659 "", tlen=18446744073709551615, t_is_prefix=0 '\000') at /data/src/10.5/strings/ctype-bin.c:87

#6  0x000055e5c675018e in my_strnncollsp_binary (cs=0x55e5c744cda0 <my_charset_bin>, s=0x7f7a2c063659 "", slen=18446744073709551615, t=0x7f7a2c063659 "", tlen=18446744073709551615) at /data/src/10.5/strings/ctype-bin.c:126

#7  0x000055e5c5dcc9ef in charset_info_st::strnncollsp (this=0x55e5c744cda0 <my_charset_bin>, a=0x7f7a2c063659 "", alen=18446744073709551615, b=0x7f7a2c063659 "", blen=18446744073709551615) at /data/src/10.5/include/m_ctype.h:782

#8  0x000055e5c5ddf4fa in SORT_FIELD_ATTR::compare_packed_varstrings (this=0x7f7a2c018350, a=0x7f7a2c063658 "", a_len=0x7f7a3e710868, b=0x7f7a2c063658 "", b_len=0x7f7a3e710870) at /data/src/10.5/sql/filesort.cc:2814

#9  0x000055e5c5ddf7a2 in compare_packed_sort_keys (sort_param=0x7f7a3e711080, a_ptr=0x7f7a2c066060, b_ptr=0x7f7a3e7108b0) at /data/src/10.5/sql/filesort.cc:2898

#10 0x000055e5c6710dcf in my_qsort2 (base_ptr=0x7f7a2c066038, count=10, size=8, cmp=0x55e5c5ddf6f9 <compare_packed_sort_keys(void*, unsigned char**, unsigned char**)>, cmp_argument=0x7f7a3e711080) at /data/src/10.5/mysys/mf_qsort.c:163

#11 0x000055e5c601ba1c in Filesort_buffer::sort_buffer (this=0x7f7a2c063250, param=0x7f7a3e711080, count=10) at /data/src/10.5/sql/filesort_utils.cc:187

#12 0x000055e5c5de03d7 in SORT_INFO::sort_buffer (this=0x7f7a2c063250, param=0x7f7a3e711080, count=10) at /data/src/10.5/sql/filesort.h:151

#13 0x000055e5c5ddb906 in save_index (param=0x7f7a3e711080, count=10, table_sort=0x7f7a2c063250) at /data/src/10.5/sql/filesort.cc:1441

#14 0x000055e5c5dd814b in filesort (thd=0x7f7a2c000b18, table=0x7f7a2c1a9498, filesort=0x7f7a2c017b70, tracker=0x7f7a2c018260, join=0x7f7a2c015668, first_table_bit=1) at /data/src/10.5/sql/filesort.cc:367

#15 0x000055e5c5b169d9 in create_sort_index (thd=0x7f7a2c000b18, join=0x7f7a2c015668, tab=0x7f7a2c016c48, fsort=0x7f7a2c017b70) at /data/src/10.5/sql/sql_select.cc:23875

#16 0x000055e5c5b10ba2 in st_join_table::sort_table (this=0x7f7a2c016c48) at /data/src/10.5/sql/sql_select.cc:21625

#17 0x000055e5c5b1077d in join_init_read_record (tab=0x7f7a2c016c48) at /data/src/10.5/sql/sql_select.cc:21564

#18 0x000055e5c5b0e525 in sub_select (join=0x7f7a2c015668, join_tab=0x7f7a2c016c48, end_of_records=false) at /data/src/10.5/sql/sql_select.cc:20638

#19 0x000055e5c5b0da40 in do_select (join=0x7f7a2c015668, procedure=0x0) at /data/src/10.5/sql/sql_select.cc:20175

#20 0x000055e5c5ae14ab in JOIN::exec_inner (this=0x7f7a2c015668) at /data/src/10.5/sql/sql_select.cc:4450

#21 0x000055e5c5ae05d7 in JOIN::exec (this=0x7f7a2c015668) at /data/src/10.5/sql/sql_select.cc:4231

#22 0x000055e5c5ae1d08 in mysql_select (thd=0x7f7a2c000b18, tables=0x7f7a2c014470, fields=..., conds=0x0, og_num=2, order=0x0, group=0x7f7a2c014bf8, having=0x0, proc_param=0x0, select_options=2147748608, result=0x7f7a2c015640, unit=0x7f7a2c004b30, select_lex=0x7f7a2c0139c0) at /data/src/10.5/sql/sql_select.cc:4655

#23 0x000055e5c5ad19ae in handle_select (thd=0x7f7a2c000b18, lex=0x7f7a2c004a68, result=0x7f7a2c015640, setup_tables_done_option=0) at /data/src/10.5/sql/sql_select.cc:429

#24 0x000055e5c5a96f00 in execute_sqlcom_select (thd=0x7f7a2c000b18, all_tables=0x7f7a2c014470) at /data/src/10.5/sql/sql_parse.cc:6209

#25 0x000055e5c5a8e1a7 in mysql_execute_command (thd=0x7f7a2c000b18) at /data/src/10.5/sql/sql_parse.cc:3931

#26 0x000055e5c5a9bd4d in mysql_parse (thd=0x7f7a2c000b18, rawbuf=0x7f7a2c0138d0 "SELECT DES_DECRYPT(a, 'x'), BINARY b FROM t1 GROUP BY 1, 2 WITH ROLLUP", length=70, parser_state=0x7f7a3e712520, is_com_multi=false, is_next_command=false) at /data/src/10.5/sql/sql_parse.cc:7993

#27 0x000055e5c5a88046 in dispatch_command (command=COM_QUERY, thd=0x7f7a2c000b18, packet=0x7f7a2c1af149 "", packet_length=70, is_com_multi=false, is_next_command=false) at /data/src/10.5/sql/sql_parse.cc:1867

#28 0x000055e5c5a8677e in do_command (thd=0x7f7a2c000b18) at /data/src/10.5/sql/sql_parse.cc:1348

#29 0x000055e5c5c2c07a in do_handle_one_connection (connect=0x55e5c92d0798, put_in_cache=true) at /data/src/10.5/sql/sql_connect.cc:1410

#30 0x000055e5c5c2bde2 in handle_one_connection (arg=0x55e5c92d0798) at /data/src/10.5/sql/sql_connect.cc:1312

#31 0x000055e5c616a126 in pfs_spawn_thread (arg=0x55e5c92d03c8) at /data/src/10.5/storage/perfschema/pfs.cc:2201

#32 0x00007f7a45aff4a4 in start_thread (arg=0x7f7a3e713700) at pthread_create.c:456

#33 0x00007f7a43c33d0f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:97

==22365==ERROR: AddressSanitizer: negative-size-param: (size=-1)

    #0 0x7ff53a035bf1  (/usr/lib/x86_64-linux-gnu/libasan.so.3+0x8dbf1)

    #1 0x556685d4a104 in my_strnncoll_binary /data/src/10.5/strings/ctype-bin.c:87

    #2 0x556685d4a187 in my_strnncollsp_binary /data/src/10.5/strings/ctype-bin.c:126

    #3 0x5566849a1faa in charset_info_st::strnncollsp(unsigned char const*, unsigned long, unsigned char const*, unsigned long) const /data/src/10.5/include/m_ctype.h:782

    #4 0x5566849caeb4 in SORT_FIELD_ATTR::compare_packed_varstrings(unsigned char*, unsigned long*, unsigned char*, unsigned long*) /data/src/10.5/sql/filesort.cc:2814

    #5 0x5566849cb572 in compare_packed_sort_keys(void*, unsigned char**, unsigned char**) /data/src/10.5/sql/filesort.cc:2898

    #6 0x556685cb3515 in my_qsort2 /data/src/10.5/mysys/mf_qsort.c:163

    #7 0x556684ee1d98 in Filesort_buffer::sort_buffer(Sort_param const*, unsigned int) /data/src/10.5/sql/filesort_utils.cc:187

    #8 0x5566849cd3aa in SORT_INFO::sort_buffer(Sort_param*, unsigned int) /data/src/10.5/sql/filesort.h:151

    #9 0x5566849c2775 in save_index /data/src/10.5/sql/filesort.cc:1441

    #10 0x5566849bb372 in filesort(THD*, TABLE*, Filesort*, Filesort_tracker*, JOIN*, unsigned long long) /data/src/10.5/sql/filesort.cc:367

    #11 0x5566843c5987 in create_sort_index(THD*, JOIN*, st_join_table*, Filesort*) /data/src/10.5/sql/sql_select.cc:23875

    #12 0x5566843b5f02 in st_join_table::sort_table() /data/src/10.5/sql/sql_select.cc:21625

    #13 0x5566843b540f in join_init_read_record(st_join_table*) /data/src/10.5/sql/sql_select.cc:21564

    #14 0x5566843af19e in sub_select(JOIN*, st_join_table*, bool) /data/src/10.5/sql/sql_select.cc:20638

    #15 0x5566843ad809 in do_select /data/src/10.5/sql/sql_select.cc:20175

    #16 0x55668434275a in JOIN::exec_inner() /data/src/10.5/sql/sql_select.cc:4450

    #17 0x55668434003f in JOIN::exec() /data/src/10.5/sql/sql_select.cc:4231

    #18 0x556684343b28 in mysql_select(THD*, TABLE_LIST*, List<Item>&, Item*, unsigned int, st_order*, st_order*, Item*, st_order*, unsigned long long, select_result*, st_select_lex_unit*, st_select_lex*) /data/src/10.5/sql/sql_select.cc:4655

    #19 0x55668431994f in handle_select(THD*, LEX*, select_result*, unsigned long) /data/src/10.5/sql/sql_select.cc:429

    #20 0x55668429a224 in execute_sqlcom_select /data/src/10.5/sql/sql_parse.cc:6209

    #21 0x55668428ab93 in mysql_execute_command(THD*) /data/src/10.5/sql/sql_parse.cc:3931

    #22 0x5566842a3f97 in mysql_parse(THD*, char*, unsigned int, Parser_state*, bool, bool) /data/src/10.5/sql/sql_parse.cc:7993

    #23 0x55668427e7f7 in dispatch_command(enum_server_command, THD*, char*, unsigned int, bool, bool) /data/src/10.5/sql/sql_parse.cc:1867

    #24 0x55668427b67a in do_command(THD*) /data/src/10.5/sql/sql_parse.cc:1348

    #25 0x556684638509 in do_handle_one_connection(CONNECT*, bool) /data/src/10.5/sql/sql_connect.cc:1410

    #26 0x556684637f78 in handle_one_connection /data/src/10.5/sql/sql_connect.cc:1312

    #27 0x556685227220 in pfs_spawn_thread /data/src/10.5/storage/perfschema/pfs.cc:2201

    #28 0x7ff539d924a3 in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x74a3)

    #29 0x7ff537ec6d0e in __clone (/lib/x86_64-linux-gnu/libc.so.6+0xe8d0e)

0x62600002a219 is located 281 bytes inside of 11084-byte region [0x62600002a100,0x62600002cc4c)

allocated by thread T5 here:

    #0 0x7ff53a069d28 in malloc (/usr/lib/x86_64-linux-gnu/libasan.so.3+0xc1d28)

    #1 0x556685d074ef in sf_malloc /data/src/10.5/mysys/safemalloc.c:118

    #2 0x556685cd8672 in my_malloc /data/src/10.5/mysys/my_malloc.c:88

    #3 0x556684ee170a in Filesort_buffer::alloc_sort_buffer(unsigned int, unsigned int) /data/src/10.5/sql/filesort_utils.cc:136

    #4 0x5566849cd40f in SORT_INFO::alloc_sort_buffer(unsigned int, unsigned int) /data/src/10.5/sql/filesort.h:160

    #5 0x5566849bb026 in filesort(THD*, TABLE*, Filesort*, Filesort_tracker*, JOIN*, unsigned long long) /data/src/10.5/sql/filesort.cc:318

    #6 0x5566843c5987 in create_sort_index(THD*, JOIN*, st_join_table*, Filesort*) /data/src/10.5/sql/sql_select.cc:23875

    #7 0x5566843b5f02 in st_join_table::sort_table() /data/src/10.5/sql/sql_select.cc:21625

    #8 0x5566843b540f in join_init_read_record(st_join_table*) /data/src/10.5/sql/sql_select.cc:21564

    #9 0x5566843af19e in sub_select(JOIN*, st_join_table*, bool) /data/src/10.5/sql/sql_select.cc:20638

    #10 0x5566843ad809 in do_select /data/src/10.5/sql/sql_select.cc:20175

    #11 0x55668434275a in JOIN::exec_inner() /data/src/10.5/sql/sql_select.cc:4450

    #12 0x55668434003f in JOIN::exec() /data/src/10.5/sql/sql_select.cc:4231

    #13 0x556684343b28 in mysql_select(THD*, TABLE_LIST*, List<Item>&, Item*, unsigned int, st_order*, st_order*, Item*, st_order*, unsigned long long, select_result*, st_select_lex_unit*, st_select_lex*) /data/src/10.5/sql/sql_select.cc:4655

    #14 0x55668431994f in handle_select(THD*, LEX*, select_result*, unsigned long) /data/src/10.5/sql/sql_select.cc:429

    #15 0x55668429a224 in execute_sqlcom_select /data/src/10.5/sql/sql_parse.cc:6209

    #16 0x55668428ab93 in mysql_execute_command(THD*) /data/src/10.5/sql/sql_parse.cc:3931

    #17 0x5566842a3f97 in mysql_parse(THD*, char*, unsigned int, Parser_state*, bool, bool) /data/src/10.5/sql/sql_parse.cc:7993

    #18 0x55668427e7f7 in dispatch_command(enum_server_command, THD*, char*, unsigned int, bool, bool) /data/src/10.5/sql/sql_parse.cc:1867

    #19 0x55668427b67a in do_command(THD*) /data/src/10.5/sql/sql_parse.cc:1348

    #20 0x556684638509 in do_handle_one_connection(CONNECT*, bool) /data/src/10.5/sql/sql_connect.cc:1410

    #21 0x556684637f78 in handle_one_connection /data/src/10.5/sql/sql_connect.cc:1312

    #22 0x556685227220 in pfs_spawn_thread /data/src/10.5/storage/perfschema/pfs.cc:2201

    #23 0x7ff539d924a3 in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x74a3)

Thread T5 created by T0 here:

    #0 0x7ff539fd8f59 in __interceptor_pthread_create (/usr/lib/x86_64-linux-gnu/libasan.so.3+0x30f59)

    #1 0x5566852222e2 in my_thread_create /data/src/10.5/storage/perfschema/my_thread.h:38

    #2 0x55668522760f in pfs_spawn_thread_v1 /data/src/10.5/storage/perfschema/pfs.cc:2252

    #3 0x556683fc3c67 in inline_mysql_thread_create /data/src/10.5/include/mysql/psi/mysql_thread.h:1321

    #4 0x556683fd6a79 in create_thread_to_handle_connection(CONNECT*) /data/src/10.5/sql/mysqld.cc:6020

    #5 0x556683fd6fee in create_new_thread(CONNECT*) /data/src/10.5/sql/mysqld.cc:6079

    #6 0x556683fd71ab in handle_accepted_socket(st_mysql_socket, st_mysql_socket) /data/src/10.5/sql/mysqld.cc:6144

    #7 0x556683fd7b8e in handle_connections_sockets() /data/src/10.5/sql/mysqld.cc:6271

    #8 0x556683fd63df in mysqld_main(int, char**) /data/src/10.5/sql/mysqld.cc:5666

    #9 0x556683fc245f in main /data/src/10.5/sql/main.cc:25

    #10 0x7ff537dfe2e0 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x202e0)

SUMMARY: AddressSanitizer: negative-size-param (/usr/lib/x86_64-linux-gnu/libasan.so.3+0x8dbf1) 

==22365==ABORTING

commit 61c15ebe323d4d6f02fab86c405b2613e5784961

Author: Monty

Date:   Thu Jul 16 16:30:06 2020 +0300

    Remove String::lex_string() and String::lex_cstring()

    - Better to use 'String *' directly.

    - Added String::get_value(LEX_STRING*) for the few cases where we want to

      convert a String to LEX_CSTRING.

    Other things:

    - Use StringBuffer for some functions to avoid mallocs