SIGFPE in Unique::get_cost_calc_buff_size from prepare_search_best_index_intersect on optimized builds

USE test;

SET SESSION OPTIMIZER_SWITCH="index_merge_sort_intersection=ON";

SET SESSION sort_buffer_size=2048;

CREATE TABLE t1(c1 VARCHAR(2049) BINARY PRIMARY KEY,c2 INT,c3 INT,INDEX(c2),UNIQUE (c1));

SELECT * FROM t1 WHERE c1>=69 AND c1<'' AND c2='';

Core was generated by `/test/MD260520-mariadb-10.5.4-linux-x86_64-opt/bin/mysqld --no-defaults --core-'.

Program terminated with signal SIGFPE, Arithmetic exception.

#0  __pthread_kill (threadid=<optimized out>, signo=signo@entry=8)

    at ../sysdeps/unix/sysv/linux/pthread_kill.c:57

[Current thread is 1 (Thread 0x15156feef700 (LWP 1371901))]

(gdb) bt

#0  __pthread_kill (threadid=<optimized out>, signo=signo@entry=8) at ../sysdeps/unix/sysv/linux/pthread_kill.c:57

#1  0x0000556280fec337 in my_write_core (sig=sig@entry=8) at /test/10.5_opt/mysys/stacktrace.c:518

#2  0x00005562809ae3ca in handle_fatal_signal (sig=8) at /test/10.5_opt/sql/signal_handler.cc:330

#3  <signal handler called>

#4  0x00005562806ab589 in Unique::get_cost_calc_buff_size (max_in_memory_size=<optimized out>, key_size=<optimized out>, nkeys=<optimized out>) at /test/10.5_opt/sql/uniques.h:89

#5  prepare_search_best_index_intersect (tree=0x151549967958, tree=0x151549967958, cutoff_cost=0.34588573959255975, init=0x15156feeb130, common=0x15156feeb1d0, param=0x15156feeb580) at /test/10.5_opt/sql/opt_range.cc:5834

#6  get_best_index_intersect (param=param@entry=0x15156feeb580, tree=tree@entry=0x151549967958, read_time=read_time@entry=0.34588573959255975) at /test/10.5_opt/sql/opt_range.cc:6349

#7  0x0000556280ad5383 in SQL_SELECT::test_quick_select (this=this@entry=0x15154984b810, thd=thd@entry=0x151549812018, keys_to_use=..., prev_tables=prev_tables@entry=0, limit=limit@entry=18446744073709551615, force_quick_range=force_quick_range@entry=false, ordered_output=false, remove_false_parts_of_where=true, only_single_index_range_scan=false) at /test/10.5_opt/sql/opt_range.cc:2951

#8  0x0000556280813810 in get_quick_record_count (limit=18446744073709551615, keys=0x15154984a6b0, table=0x151549912e18, select=0x15154984b810, thd=0x151549812018) at /test/10.5_opt/sql/sql_select.cc:4713

#9  make_join_statistics (keyuse_array=0x151549849818, tables_list=..., join=0x151549849528) at /test/10.5_opt/sql/sql_select.cc:5438

#10 JOIN::optimize_inner (this=this@entry=0x151549849528) at /test/10.5_opt/sql/sql_select.cc:2260

#11 0x000055628081486b in JOIN::optimize (this=this@entry=0x151549849528) at /test/10.5_opt/sql/sql_select.cc:1606

#12 0x0000556280814971 in mysql_select (thd=thd@entry=0x151549812018, tables=0x1515498476f0, fields=..., conds=0x151549848620, og_num=<optimized out>, order=0x0, group=0x0, having=0x0, proc_param=0x0, select_options=2147748608, result=0x151549849500, unit=0x151549815e70, select_lex=0x1515498470f8) at /test/10.5_opt/sql/sql_select.cc:4655

#13 0x0000556280815381 in handle_select (thd=thd@entry=0x151549812018, lex=lex@entry=0x151549815da8, result=result@entry=0x151549849500, setup_tables_done_option=setup_tables_done_option@entry=0) at /test/10.5_opt/sql/sql_select.cc:417

#14 0x00005562807bbe91 in execute_sqlcom_select (thd=thd@entry=0x151549812018, all_tables=0x1515498476f0) at /test/10.5_opt/sql/sql_parse.cc:6207

#15 0x00005562807b7db2 in mysql_execute_command (thd=thd@entry=0x151549812018) at /test/10.5_opt/sql/sql_parse.cc:3939

#16 0x00005562807befac in mysql_parse (thd=0x151549812018, rawbuf=<optimized out>, length=49, parser_state=0x15156feee4b0, is_com_multi=<optimized out>, is_next_command=<optimized out>) at /test/10.5_opt/sql/sql_parse.cc:7991

#17 0x00005562807b42b5 in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x151549812018, packet=packet@entry=0x15154983a019 "SELECT * FROM t1 WHERE c1>=69 AND c1<'' AND c2=''", packet_length=packet_length@entry=49, is_com_multi=is_com_multi@entry=false, is_next_command=is_next_command@entry=false) at /test/10.5_opt/sql/sql_parse.cc:1874

#18 0x00005562807b26a4 in do_command (thd=0x151549812018) at /test/10.5_opt/sql/sql_parse.cc:1355

#19 0x00005562808a7891 in do_handle_one_connection (connect=<optimized out>, connect@entry=0x15156d8329b8, put_in_cache=put_in_cache@entry=true) at /test/10.5_opt/sql/sql_connect.cc:1411

#20 0x00005562808a7bf4 in handle_one_connection (arg=arg@entry=0x15156d8329b8) at /test/10.5_opt/sql/sql_connect.cc:1313

#21 0x0000556280c1406a in pfs_spawn_thread (arg=0x15156d84b018) at /test/10.5_opt/storage/perfschema/pfs.cc:2201

#22 0x000015156f3166db in start_thread (arg=0x15156feef700) at pthread_create.c:463

#23 0x000015156e71488f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

10.1.46>CREATE TABLE t1(c1 VARCHAR(2049) BINARY PRIMARY KEY,c2 INT,c3 INT,INDEX(c2),UNIQUE (c1));

ERROR 1071 (42000): Specified key was too long; max key length is 767 bytes