Server crashes when using RANDOM_BYTES function and GROUP BY clause on a column with a negative value

--source include/have_innodb.inc

SET sql_mode='';

CREATE OR REPLACE TABLE t (a VARCHAR(255)) ENGINE=InnoDB;

INSERT INTO t VALUES (9494),(9495),(9496),(9497),(9498),(9499),(9500),(9501),(9502),(9503);

SELECT RANDOM_BYTES (-1) f1,a f2 FROM t GROUP BY f1,f2;

Core was generated by `/test/MD010224-mariadb-11.3.2-linux-x86_64-opt/bin/mariadbd --no-defaults --max'.

Program terminated with signal SIGSEGV, Segmentation fault.

#0  __memcmp_avx2_movbe ()

    at ../sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S:314

[Current thread is 1 (Thread 0x148f880a5700 (LWP 3093479))]

(gdb) bt

#0  __memcmp_avx2_movbe () at ../sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S:314

#1  0x000055f2cdda94eb in my_strnncoll_binary (t_is_prefix=0 '\000', tlen=18446744073709551615, t=<optimized out>, slen=18446744073709551615, s=<optimized out>, cs=<optimized out>) at /test/11.3_opt/strings/ctype-bin.c:89

#2  my_strnncollsp_binary (cs=<optimized out>, s=<optimized out>, slen=18446744073709551615, t=<optimized out>, tlen=18446744073709551615) at /test/11.3_opt/strings/ctype-bin.c:128

#3  0x000055f2cd831a0b in charset_info_st::strnncollsp (blen=<optimized out>, b=<optimized out>, alen=<optimized out>, a=<optimized out>, this=<optimized out>) at /test/11.3_opt/include/m_ctype.h:1017

#4  SORT_FIELD_ATTR::compare_packed_varstrings (this=this@entry=0x148f44049680, a=a@entry=0x148f4408d136 "", a_len=a_len@entry=0x148f880a27d0, b=b@entry=0x148f4408d136 "", b_len=b_len@entry=0x148f880a27d8) at /test/11.3_opt/sql/filesort.cc:2766

#5  0x000055f2cd831bff in compare_packed_sort_keys (sort_param=0x148f880a2ef0, a_ptr=<optimized out>, b_ptr=<optimized out>) at /test/11.3_opt/sql/sql_class.h:7166

#6  0x000055f2cdd7f373 in my_qsort2 (base_ptr=<optimized out>, count=<optimized out>, count@entry=10, size=size@entry=8, cmp=<optimized out>, cmp_argument=cmp_argument@entry=0x148f880a2ef0) at /test/11.3_opt/mysys/mf_qsort.c:163

#7  0x000055f2cd82c45c in Filesort_buffer::sort_buffer (this=this@entry=0x148f4408cf40, param=param@entry=0x148f880a2ef0, count=count@entry=10) at /test/11.3_opt/sql/sql_sort.h:693

#8  0x000055f2cd8313b7 in SORT_INFO::sort_buffer (count=10, param=0x148f880a2ef0, this=0x148f4408cf40) at /test/11.3_opt/sql/filesort.h:168

#9  save_index (table_sort=0x148f4408cf40, count=10, param=0x148f880a2ef0) at /test/11.3_opt/sql/filesort.cc:1521

#10 filesort (thd=0x148f44000c58, table=table@entry=0x148f4404c3e0, filesort=filesort@entry=0x148f440147b0, tracker=0x148f44014980, join=join@entry=0x148f440128c0, first_table_bit=<optimized out>) at /test/11.3_opt/sql/filesort.cc:423

#11 0x000055f2cd61a622 in create_sort_index (thd=<optimized out>, join=0x148f440128c0, tab=tab@entry=0x148f44048830, fsort=0x148f440147b0, fsort@entry=0x0) at /test/11.3_opt/sql/sql_select.cc:27072

#12 0x000055f2cd61a9d2 in st_join_table::sort_table (this=this@entry=0x148f44048830) at /test/11.3_opt/sql/sql_select.cc:24686

#13 0x000055f2cd61aaa5 in join_init_read_record (tab=0x148f44048830) at /test/11.3_opt/sql/sql_select.cc:24606

#14 0x000055f2cd62ae05 in AGGR_OP::end_send (this=0x148f44014660) at /test/11.3_opt/sql/sql_select.cc:32556

#15 0x000055f2cd62b0f9 in sub_select_postjoin_aggr (join=0x148f440128c0, join_tab=0x148f44048830, end_of_records=<optimized out>) at /test/11.3_opt/sql/sql_select.cc:23282

#16 0x000055f2cd634b02 in do_select (procedure=<optimized out>, join=0x148f440128c0) at /test/11.3_opt/sql/sql_select.cc:23117

#17 JOIN::exec_inner (this=0x148f440128c0) at /test/11.3_opt/sql/sql_select.cc:4988

#18 0x000055f2cd6350ce in JOIN::exec (this=this@entry=0x148f440128c0) at /test/11.3_opt/sql/sql_select.cc:4774

#19 0x000055f2cd633110 in mysql_select (thd=0x148f44000c58, tables=0x148f44011568, fields=<optimized out>, conds=0x0, og_num=2, order=0x0, group=0x148f44011db8, having=0x0, proc_param=0x0, select_options=<optimized out>, result=0x148f44012898, unit=0x148f44004f10, select_lex=0x148f44010bd0) at /test/11.3_opt/sql/sql_select.cc:5304

#20 0x000055f2cd633917 in handle_select (thd=thd@entry=0x148f44000c58, lex=lex@entry=0x148f44004e30, result=result@entry=0x148f44012898, setup_tables_done_option=setup_tables_done_option@entry=0) at /test/11.3_opt/sql/sql_select.cc:630

#21 0x000055f2cd5ad32e in execute_sqlcom_select (thd=0x148f44000c58, all_tables=0x148f44011568) at /test/11.3_opt/sql/sql_parse.cc:6077

#22 0x000055f2cd5bb961 in mysql_execute_command (thd=0x148f44000c58, is_called_from_prepared_stmt=<optimized out>) at /test/11.3_opt/sql/sql_parse.cc:3926

#23 0x000055f2cd5a7ef6 in mysql_parse (thd=0x148f44000c58, rawbuf=<optimized out>, length=<optimized out>, parser_state=<optimized out>) at /test/11.3_opt/sql/sql_parse.cc:7798

#24 0x000055f2cd5b4475 in dispatch_command (command=COM_QUERY, thd=0x148f44000c58, packet=<optimized out>, packet_length=<optimized out>, blocking=<optimized out>) at /test/11.3_opt/sql/sql_class.h:1550

#25 0x000055f2cd5b66be in do_command (thd=0x148f44000c58, blocking=blocking@entry=true) at /test/11.3_opt/sql/sql_parse.cc:1406

#26 0x000055f2cd6e6197 in do_handle_one_connection (connect=<optimized out>, put_in_cache=true) at /test/11.3_opt/sql/sql_connect.cc:1417

#27 0x000055f2cd6e64dd in handle_one_connection (arg=arg@entry=0x55f2d0db8cf8) at /test/11.3_opt/sql/sql_connect.cc:1319

#28 0x000055f2cda8f0bc in pfs_spawn_thread (arg=0x55f2d0de08e8) at /test/11.3_opt/storage/perfschema/pfs.cc:2201

#29 0x0000148f8b3e2609 in start_thread (arg=<optimized out>) at pthread_create.c:477

#30 0x0000148f8afce133 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

mariadbd: /test/11.4_dbg/sql/filesort.cc:2522: virtual uint Type_handler_string_result::make_packed_sort_key_part(uchar*, Item*, const SORT_FIELD_ATTR*, String*) const: Assertion `0' failed.

Core was generated by `/test/MD010224-mariadb-11.4.0-linux-x86_64-dbg/bin/mariadbd --no-defaults --max'.

Program terminated with signal SIGABRT, Aborted.

#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50

[Current thread is 1 (Thread 0x1510980d9700 (LWP 3093565))]

(gdb) bt

#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50

#1  0x000015109b53f859 in __GI_abort () at abort.c:79

#2  0x000015109b53f729 in __assert_fail_base (fmt=0x15109b6d5588 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=0x55979bd05ca8 "0", file=0x55979bce4a70 "/test/11.4_dbg/sql/filesort.cc", line=2522, function=<optimized out>) at assert.c:92

#3  0x000015109b550fd6 in __GI___assert_fail (assertion=assertion@entry=0x55979bd05ca8 "0", file=file@entry=0x55979bce4a70 "/test/11.4_dbg/sql/filesort.cc", line=line@entry=2522, function=function@entry=0x55979bce5c18 "virtual uint Type_handler_string_result::make_packed_sort_key_part(uchar*, Item*, const SORT_FIELD_ATTR*, String*) const") at assert.c:101

#4  0x000055979b1bdbd6 in Type_handler_string_result::make_packed_sort_key_part (this=<optimized out>, to=0x15104c0b20cc '\245' <repeats 200 times>..., item=<optimized out>, sort_field=0x15104c06e3e8, tmp=<optimized out>) at /test/11.4_dbg/sql/filesort.cc:2522

#5  0x000055979b1b9431 in make_packed_sortkey (to=0x15104c0b20cc '\245' <repeats 200 times>..., param=0x1510980d6900) at /test/11.4_dbg/sql/filesort.cc:3010

#6  make_sortkey (param=param@entry=0x1510980d6900, to=0x15104c0b20c8 '\245' <repeats 200 times>..., ref_pos=ref_pos@entry=0x15104c072190 "\310<\aL\020\025", using_packed_sortkeys=using_packed_sortkeys@entry=true) at /test/11.4_dbg/sql/filesort.cc:1414

#7  0x000055979b1bc71d in find_all_keys (found_rows=0x15104c0b20a0, pq=0x0, tempfile=0x1510980d69c0, buffpek_pointers=0x1510980d6b30, fs_info=0x15104c0b1eb0, select=0x0, param=0x1510980d6900, thd=0x15104c000d48) at /test/11.4_dbg/sql/filesort.cc:1031

#8  filesort (thd=thd@entry=0x15104c000d48, table=table@entry=0x15104c071040, filesort=filesort@entry=0x15104c0172f8, tracker=0x15104c06ddb0, join=join@entry=0x15104c0152c0, first_table_bit=<optimized out>) at /test/11.4_dbg/sql/filesort.cc:408

#9  0x000055979af15064 in create_sort_index (thd=0x15104c000d48, join=0x15104c0152c0, tab=tab@entry=0x15104c06d490, fsort=0x15104c0172f8, fsort@entry=0x0) at /test/11.4_dbg/sql/sql_select.cc:27072

#10 0x000055979af152d9 in st_join_table::sort_table (this=this@entry=0x15104c06d490) at /test/11.4_dbg/sql/sql_select.cc:24686

#11 0x000055979af153f9 in join_init_read_record (tab=0x15104c06d490) at /test/11.4_dbg/sql/sql_select.cc:24606

#12 0x000055979af2733b in AGGR_OP::end_send (this=this@entry=0x15104c0171a8) at /test/11.4_dbg/sql/sql_select.cc:32556

#13 0x000055979af276f0 in sub_select_postjoin_aggr (join=0x15104c0152c0, join_tab=0x15104c06d490, end_of_records=<optimized out>) at /test/11.4_dbg/sql/sql_select.cc:23282

#14 0x000055979aef96c0 in sub_select (join=0x15104c0152c0, join_tab=0x15104c06d020, end_of_records=true) at /test/11.4_dbg/sql/sql_select.cc:23536

#15 0x000055979af34254 in do_select (procedure=<optimized out>, join=0x15104c0152c0) at /test/11.4_dbg/sql/sql_select.cc:23117

#16 JOIN::exec_inner (this=this@entry=0x15104c0152c0) at /test/11.4_dbg/sql/sql_select.cc:4988

#17 0x000055979af34792 in JOIN::exec (this=this@entry=0x15104c0152c0) at /test/11.4_dbg/sql/sql_select.cc:4774

#18 0x000055979af325dc in mysql_select (thd=thd@entry=0x15104c000d48, tables=0x15104c013f68, fields=@0x15104c013888: {<base_list> = {<Sql_alloc> = {<No data fields>}, first = 0x15104c013c38, last = 0x15104c013f18, elements = 2}, <No data fields>}, conds=0x0, og_num=2, order=0x0, group=0x15104c0147b8, having=0x0, proc_param=0x0, select_options=2164525824, result=0x15104c015298, unit=0x15104c0051c8, select_lex=0x15104c0135d0) at /test/11.4_dbg/sql/sql_select.cc:5304

#19 0x000055979af32e05 in handle_select (thd=thd@entry=0x15104c000d48, lex=lex@entry=0x15104c0050e8, result=result@entry=0x15104c015298, setup_tables_done_option=setup_tables_done_option@entry=0) at /test/11.4_dbg/sql/sql_select.cc:630

#20 0x000055979ae91f22 in execute_sqlcom_select (thd=thd@entry=0x15104c000d48, all_tables=0x15104c013f68) at /test/11.4_dbg/sql/sql_parse.cc:6077

#21 0x000055979ae9ea1b in mysql_execute_command (thd=thd@entry=0x15104c000d48, is_called_from_prepared_stmt=is_called_from_prepared_stmt@entry=false) at /test/11.4_dbg/sql/sql_parse.cc:3926

#22 0x000055979ae8be9e in mysql_parse (thd=thd@entry=0x15104c000d48, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x1510980d82b0) at /test/11.4_dbg/sql/sql_parse.cc:7798

#23 0x000055979ae99d7a in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x15104c000d48, packet=packet@entry=0x15104c00b1b9 "", packet_length=packet_length@entry=64, blocking=blocking@entry=true) at /test/11.4_dbg/sql/sql_class.h:1552

#24 0x000055979ae9c4a6 in do_command (thd=0x15104c000d48, blocking=blocking@entry=true) at /test/11.4_dbg/sql/sql_parse.cc:1406

#25 0x000055979b0176b3 in do_handle_one_connection (connect=<optimized out>, connect@entry=0x55979f0ada18, put_in_cache=put_in_cache@entry=true) at /test/11.4_dbg/sql/sql_connect.cc:1417

#26 0x000055979b017c68 in handle_one_connection (arg=arg@entry=0x55979f0ada18) at /test/11.4_dbg/sql/sql_connect.cc:1319

#27 0x000055979b48b1da in pfs_spawn_thread (arg=0x55979f054538) at /test/11.4_dbg/storage/perfschema/pfs.cc:2201

#28 0x000015109ba50609 in start_thread (arg=<optimized out>) at pthread_create.c:477

#29 0x000015109b63c133 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95