Server crashes in Charset::numchars / String::numchars

SELECT NULL IN (RIGHT(AES_ENCRYPT('foo','bar'), LAST_INSERT_ID()), 'qux') ;

#3  <signal handler called>

#4  String::numchars (this=0x7fd3e0013928) at /data/src/10.2/sql/sql_string.cc:630

#5  0x000055960d2729f9 in Item_func_right::val_str (this=0x7fd3e0012d28, str=0x7fd3e0013928) at /data/src/10.2/sql/item_strfunc.cc:1625

#6  0x000055960d20ae59 in in_string::set (this=0x7fd3e0013888, pos=0, item=0x7fd3e0012d28) at /data/src/10.2/sql/item_cmpfunc.cc:3678

#7  0x000055960d20ccac in Item_func_in::create_array (this=0x7fd3e0012ef0, thd=0x7fd3e0000d90) at /data/src/10.2/sql/item_cmpfunc.cc:4214

#8  0x000055960d20d42a in Item_func_in::fix_length_and_dec (this=0x7fd3e0012ef0) at /data/src/10.2/sql/item_cmpfunc.cc:4372

#9  0x000055960d23b504 in Item_func::fix_fields (this=0x7fd3e0012ef0, thd=0x7fd3e0000d90, ref=0x7fd3e0013048) at /data/src/10.2/sql/item_func.cc:230

#10 0x000055960d20c7ab in Item_func_in::fix_fields (this=0x7fd3e0012ef0, thd=0x7fd3e0000d90, ref=0x7fd3e0013048) at /data/src/10.2/sql/item_cmpfunc.cc:4125

#11 0x000055960ced571b in setup_fields (thd=0x7fd3e0000d90, ref_pointer_array=..., fields=..., mark_used_columns=MARK_COLUMNS_READ, sum_func_list=0x7fd3e00134e8, pre_fix=0x7fd3e0005208, allow_sum_func=true) at /data/src/10.2/sql/sql_base.cc:7283

#12 0x000055960cf7df6f in JOIN::prepare (this=0x7fd3e00131c8, tables_init=0x0, wild_num=0, conds_init=0x0, og_num=0, order_init=0x0, skip_order_by=false, group_init=0x0, having_init=0x0, proc_param_init=0x0, select_lex_arg=0x7fd3e00050c8, unit_arg=0x7fd3e0004988) at /data/src/10.2/sql/sql_select.cc:814

#13 0x000055960cf88986 in mysql_select (thd=0x7fd3e0000d90, tables=0x0, wild_num=0, fields=..., conds=0x0, og_num=0, order=0x0, group=0x0, having=0x0, proc_param=0x0, select_options=2147748608, result=0x7fd3e00131a8, unit=0x7fd3e0004988, select_lex=0x7fd3e00050c8) at /data/src/10.2/sql/sql_select.cc:3814

#14 0x000055960cf7cbb2 in handle_select (thd=0x7fd3e0000d90, lex=0x7fd3e00048c8, result=0x7fd3e00131a8, setup_tables_done_option=0) at /data/src/10.2/sql/sql_select.cc:361

#15 0x000055960cf472bc in execute_sqlcom_select (thd=0x7fd3e0000d90, all_tables=0x0) at /data/src/10.2/sql/sql_parse.cc:6248

#16 0x000055960cf3dc6b in mysql_execute_command (thd=0x7fd3e0000d90) at /data/src/10.2/sql/sql_parse.cc:3559

#17 0x000055960cf4b065 in mysql_parse (thd=0x7fd3e0000d90, rawbuf=0x7fd3e0012840 "SELECT NULL IN (RIGHT(AES_ENCRYPT('foo','bar'), LAST_INSERT_ID()), 'qux')", length=73, parser_state=0x7fd3fc6c15f0, is_com_multi=false, is_next_command=false) at /data/src/10.2/sql/sql_parse.cc:7763

#18 0x000055960cf3933e in dispatch_command (command=COM_QUERY, thd=0x7fd3e0000d90, packet=0x7fd3e0008b51 "SELECT NULL IN (RIGHT(AES_ENCRYPT('foo','bar'), LAST_INSERT_ID()), 'qux') ", packet_length=74, is_com_multi=false, is_next_command=false) at /data/src/10.2/sql/sql_parse.cc:1827

#19 0x000055960cf37e39 in do_command (thd=0x7fd3e0000d90) at /data/src/10.2/sql/sql_parse.cc:1381

#20 0x000055960d092754 in do_handle_one_connection (connect=0x55960fd8d9c0) at /data/src/10.2/sql/sql_connect.cc:1336

#21 0x000055960d0924b9 in handle_one_connection (arg=0x55960fd8d9c0) at /data/src/10.2/sql/sql_connect.cc:1241

#22 0x000055960d8baa48 in pfs_spawn_thread (arg=0x55960fd70dd0) at /data/src/10.2/storage/perfschema/pfs.cc:1869

#23 0x00007fd402276609 in start_thread (arg=<optimized out>) at pthread_create.c:477

#24 0x00007fd401e52293 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

#3  <signal handler called>

#4  0x000055e4efd093e6 in charset_info_st::numchars (this=0x0, b=0x7fbc8c1b28b8 "\223\362T\222H\001\270\260\360", e=0x7fbc8c1b28c8 "\245\245\245\245\245\245\245\245h4z\025\217\217\217\217\305\001") at /data/src/10.6/include/m_ctype.h:608

#5  0x000055e4efd09598 in Charset::numchars (this=0x7fbc8c015c38, str=0x7fbc8c1b28b8 "\223\362T\222H\001\270\260\360", end=0x7fbc8c1b28c8 "\245\245\245\245\245\245\245\245h4z\025\217\217\217\217\305\001") at /data/src/10.6/sql/sql_string.h:162

#6  0x000055e4efd09602 in String::numchars (this=0x7fbc8c015c38) at /data/src/10.6/sql/sql_string.h:951

#7  0x000055e4f020379c in Item_func_right::val_str (this=0x7fbc8c014798, str=0x7fbc8c015c38) at /data/src/10.6/sql/item_strfunc.cc:1675

#8  0x000055e4f01883d3 in in_string::set (this=0x7fbc8c015b98, pos=0, item=0x7fbc8c014798) at /data/src/10.6/sql/item_cmpfunc.cc:3688

#9  0x000055e4f018b4f8 in Item_func_in::fix_in_vector (this=0x7fbc8c014970) at /data/src/10.6/sql/item_cmpfunc.cc:4451

#10 0x000055e4f002de5a in Item_func_in::fix_for_scalar_comparison_using_bisection (this=0x7fbc8c014970, thd=0x7fbc8c000db8) at /data/src/10.6/sql/item_cmpfunc.h:2423

#11 0x000055e4f001611d in Type_handler_string_result::Item_func_in_fix_comparator_compatible_types (this=0x55e4f172a640 <type_handler_long_blob>, thd=0x7fbc8c000db8, func=0x7fbc8c014970) at /data/src/10.6/sql/sql_type.cc:5771

#12 0x000055e4f018b2b8 in Item_func_in::fix_length_and_dec (this=0x7fbc8c014970) at /data/src/10.6/sql/item_cmpfunc.cc:4418

#13 0x000055e4f01bbb90 in Item_func::fix_fields (this=0x7fbc8c014970, thd=0x7fbc8c000db8, ref=0x7fbc8c014af8) at /data/src/10.6/sql/item_func.cc:370

#14 0x000055e4f018ae07 in Item_func_in::fix_fields (this=0x7fbc8c014970, thd=0x7fbc8c000db8, ref=0x7fbc8c014af8) at /data/src/10.6/sql/item_cmpfunc.cc:4334

#15 0x000055e4efca0aa5 in Item::fix_fields_if_needed (this=0x7fbc8c014970, thd=0x7fbc8c000db8, ref=0x7fbc8c014af8) at /data/src/10.6/sql/item.h:988

#16 0x000055e4efca0adf in Item::fix_fields_if_needed_for_scalar (this=0x7fbc8c014970, thd=0x7fbc8c000db8, ref=0x7fbc8c014af8) at /data/src/10.6/sql/item.h:992

#17 0x000055e4efd2502f in setup_fields (thd=0x7fbc8c000db8, ref_pointer_array=..., fields=..., column_usage=MARK_COLUMNS_READ, sum_func_list=0x7fbc8c0157d0, pre_fix=0x7fbc8c014088, allow_sum_func=true) at /data/src/10.6/sql/sql_base.cc:7648

#18 0x000055e4efe0f8b3 in JOIN::prepare (this=0x7fbc8c0154a8, tables_init=0x0, conds_init=0x0, og_num=0, order_init=0x0, skip_order_by=false, group_init=0x0, having_init=0x0, proc_param_init=0x0, select_lex_arg=0x7fbc8c013f20, unit_arg=0x7fbc8c004f80) at /data/src/10.6/sql/sql_select.cc:1259

#19 0x000055e4efe1c50a in mysql_select (thd=0x7fbc8c000db8, tables=0x0, fields=..., conds=0x0, og_num=0, order=0x0, group=0x0, having=0x0, proc_param=0x0, select_options=2147748608, result=0x7fbc8c015480, unit=0x7fbc8c004f80, select_lex=0x7fbc8c013f20) at /data/src/10.6/sql/sql_select.cc:4648

#20 0x000055e4efe0c149 in handle_select (thd=0x7fbc8c000db8, lex=0x7fbc8c004eb8, result=0x7fbc8c015480, setup_tables_done_option=0) at /data/src/10.6/sql/sql_select.cc:417

#21 0x000055e4efdceac2 in execute_sqlcom_select (thd=0x7fbc8c000db8, all_tables=0x0) at /data/src/10.6/sql/sql_parse.cc:6138

#22 0x000055e4efdc5b48 in mysql_execute_command (thd=0x7fbc8c000db8) at /data/src/10.6/sql/sql_parse.cc:3834

#23 0x000055e4efdd38f8 in mysql_parse (thd=0x7fbc8c000db8, rawbuf=0x7fbc8c013e30 "SELECT NULL IN (RIGHT(AES_ENCRYPT('foo','bar'), LAST_INSERT_ID()), 'qux')", length=73, parser_state=0x7fbc9d34b510) at /data/src/10.6/sql/sql_parse.cc:7906

#24 0x000055e4efdbfe3f in dispatch_command (command=COM_QUERY, thd=0x7fbc8c000db8, packet=0x7fbc8c008e49 "SELECT NULL IN (RIGHT(AES_ENCRYPT('foo','bar'), LAST_INSERT_ID()), 'qux') ", packet_length=74) at /data/src/10.6/sql/sql_parse.cc:1833

#25 0x000055e4efdbe856 in do_command (thd=0x7fbc8c000db8) at /data/src/10.6/sql/sql_parse.cc:1365

#26 0x000055e4eff6bfa9 in do_handle_one_connection (connect=0x55e4f3d1f848, put_in_cache=true) at /data/src/10.6/sql/sql_connect.cc:1410

#27 0x000055e4eff6bd0c in handle_one_connection (arg=0x55e4f3c92b68) at /data/src/10.6/sql/sql_connect.cc:1312

#28 0x000055e4f04c8cc7 in pfs_spawn_thread (arg=0x55e4f3d20ea8) at /data/src/10.6/storage/perfschema/pfs.cc:2201

#29 0x00007fbca2ea8609 in start_thread (arg=<optimized out>) at pthread_create.c:477

#30 0x00007fbca2a7c293 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

commit 3a37afec293e36e51b83a9bd338ad5f74e7f63c0

Author: Alexander Barkov

Date:   Mon Jun 19 12:45:32 2017 +0400

    MDEV-10306 Wrong results with combination of CONCAT, SUBSTR and CONVERT in subquery