Server crash or assertion failure in _my_b_write / merge_buffers upon UPDATE with subquery and prefix key

--source include/have_sequence.inc

CREATE TABLE t1 (a INT, b CHAR(8), KEY(a), KEY(b(3)));

INSERT INTO t1 (a) SELECT seq FROM seq_1_to_8700;

CREATE TABLE t2 (f INT);

INSERT INTO t2 VALUES (1),(1);

UPDATE t2 SET f = 0 WHERE f NOT IN (SELECT 1 FROM t1 WHERE a = 72 OR b IS NULL);

# Cleanup

DROP TABLE t2, t1;

mysqld: /data/src/10.2/mysys/mf_iocache.c:588: _my_b_write: Assertion `Count >= rest_length' failed.

210530 18:17:12 [ERROR] mysqld got signal 6 ;

#7  0x00007f58643d1f36 in __GI___assert_fail (assertion=0x55b8bd6d1527 "Count >= rest_length", file=0x55b8bd6d1300 "/data/src/10.2/mysys/mf_iocache.c", line=588, function=0x55b8bd6d1a10 <__PRETTY_FUNCTION__.14504> "_my_b_write") at assert.c:101

#8  0x000055b8bd36eb7b in _my_b_write (info=0x7f584c092068, Buffer=0x7f584c0e56cf "", Count=7) at /data/src/10.2/mysys/mf_iocache.c:588

#9  0x000055b8bcc31e22 in my_b_write (info=0x7f584c092068, Buffer=0x7f584c0e56cf "", Count=7) at /data/src/10.2/include/my_sys.h:544

#10 0x000055b8bcc36b45 in merge_buffers (param=0x7f585e3dcb60, from_file=0x7f584c091c20, to_file=0x7f584c092068, sort_buffer=0x7f584c0a56d0 "", lastbuff=0x7f584c094cc0, Fb=0x7f584c094cc0, Tb=0x7f584c094cf0, flag=1) at /data/src/10.2/sql/filesort.cc:1734

#11 0x000055b8bcc37103 in merge_index (param=0x7f585e3dcb60, sort_buffer=0x7f584c0a56d0 "", buffpek=0x7f584c094cc0, maxbuffer=1, tempfile=0x7f584c091c20, outfile=0x7f584c092068) at /data/src/10.2/sql/filesort.cc:1866

#12 0x000055b8bcadeddf in Unique::merge (this=0x7f584c091bf0, table=0x7f584c17e9c0, buff=0x7f584c0a56d0 "", without_last_merge=false) at /data/src/10.2/sql/uniques.cc:742

#13 0x000055b8bcadf097 in Unique::get (this=0x7f584c091bf0, table=0x7f584c17e9c0) at /data/src/10.2/sql/uniques.cc:802

#14 0x000055b8bcda0e25 in read_keys_and_merge_scans (thd=0x7f584c000d90, head=0x7f584c17e9c0, quick_selects=..., pk_quick_select=0x0, read_record=0x7f584c17d138, intersection=false, filtered_scans=0x0, unique_ptr=0x7f584c17d0d0) at /data/src/10.2/sql/opt_range.cc:11105

#15 0x000055b8bcda0f99 in QUICK_INDEX_MERGE_SELECT::read_keys_and_merge (this=0x7f584c17d090) at /data/src/10.2/sql/opt_range.cc:11126

#16 0x000055b8bcd8a1fd in QUICK_INDEX_SORT_SELECT::reset (this=0x7f584c17d090) at /data/src/10.2/sql/opt_range.cc:1365

#17 0x000055b8bca28bb1 in join_init_read_record (tab=0x7f584c017988) at /data/src/10.2/sql/sql_select.cc:19785

#18 0x000055b8bca26a2c in sub_select (join=0x7f584c014e60, join_tab=0x7f584c017988, end_of_records=false) at /data/src/10.2/sql/sql_select.cc:18871

#19 0x000055b8bca25ff0 in do_select (join=0x7f584c014e60, procedure=0x0) at /data/src/10.2/sql/sql_select.cc:18418

#20 0x000055b8bc9ffbd1 in JOIN::exec_inner (this=0x7f584c014e60) at /data/src/10.2/sql/sql_select.cc:3651

#21 0x000055b8bc9ff078 in JOIN::exec (this=0x7f584c014e60) at /data/src/10.2/sql/sql_select.cc:3446

#22 0x000055b8bcd0ec23 in subselect_single_select_engine::exec (this=0x7f584c014b80) at /data/src/10.2/sql/item_subselect.cc:4000

#23 0x000055b8bcd0356d in Item_subselect::exec (this=0x7f584c014980) at /data/src/10.2/sql/item_subselect.cc:770

#24 0x000055b8bcd03ba6 in Item_in_subselect::exec (this=0x7f584c014980) at /data/src/10.2/sql/item_subselect.cc:950

#25 0x000055b8bcd06d2f in Item_in_subselect::val_bool (this=0x7f584c014980) at /data/src/10.2/sql/item_subselect.cc:1855

#26 0x000055b8bc8da7a1 in Item::val_bool_result (this=0x7f584c014980) at /data/src/10.2/sql/item.h:1278

#27 0x000055b8bcc7d33b in Item_in_optimizer::val_int (this=0x7f584c015478) at /data/src/10.2/sql/item_cmpfunc.cc:1677

#28 0x000055b8bcc4e359 in Item::val_bool (this=0x7f584c015478) at /data/src/10.2/sql/item.cc:112

#29 0x000055b8bcc78d5d in Item_func_not::val_int (this=0x7f584c014bc0) at /data/src/10.2/sql/item_cmpfunc.cc:307

#30 0x000055b8bcab0e51 in SQL_SELECT::skip_record (this=0x7f584c0917a0, thd=0x7f584c000d90) at /data/src/10.2/sql/opt_range.h:1633

#31 0x000055b8bcaaaab9 in mysql_update (thd=0x7f584c000d90, table_list=0x7f584c012848, fields=..., values=..., conds=0x7f584c014bc0, order_num=0, order=0x0, limit=18446744073709551615, handle_duplicates=DUP_ERROR, ignore=false, found_return=0x7f585e3ddbd0, updated_return=0x7f585e3ddc80) at /data/src/10.2/sql/sql_update.cc:755

#32 0x000055b8bc9b6c9d in mysql_execute_command (thd=0x7f584c000d90) at /data/src/10.2/sql/sql_parse.cc:4056

#33 0x000055b8bc9c2856 in mysql_parse (thd=0x7f584c000d90, rawbuf=0x7f584c012708 "UPDATE t2 SET f = 0 WHERE f NOT IN (SELECT 1 FROM t1 WHERE a = 72 OR b IS NULL)", length=79, parser_state=0x7f585e3de560, is_com_multi=false, is_next_command=false) at /data/src/10.2/sql/sql_parse.cc:7793

#34 0x000055b8bc9b0ab1 in dispatch_command (command=COM_QUERY, thd=0x7f584c000d90, packet=0x7f584c008b61 "UPDATE t2 SET f = 0 WHERE f NOT IN (SELECT 1 FROM t1 WHERE a = 72 OR b IS NULL)", packet_length=79, is_com_multi=false, is_next_command=false) at /data/src/10.2/sql/sql_parse.cc:1827

#35 0x000055b8bc9af5ac in do_command (thd=0x7f584c000d90) at /data/src/10.2/sql/sql_parse.cc:1381

#36 0x000055b8bcb0aa31 in do_handle_one_connection (connect=0x55b8be96e110) at /data/src/10.2/sql/sql_connect.cc:1336

#37 0x000055b8bcb0a796 in handle_one_connection (arg=0x55b8be96e110) at /data/src/10.2/sql/sql_connect.cc:1241

#38 0x000055b8bd336ab0 in pfs_spawn_thread (arg=0x55b8be951440) at /data/src/10.2/storage/perfschema/pfs.cc:1869

#39 0x00007f58648e1609 in start_thread (arg=<optimized out>) at pthread_create.c:477

#40 0x00007f58644bd293 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

#19 0x0000560c0b5a032b in handle_fatal_signal (sig=11) at /data/src/10.2/sql/signal_handler.cc:218

#20 <signal handler called>

#21 __memmove_avx_unaligned_erms () at ../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S:497

#22 0x0000560c0badfab5 in memcpy (__len=18446744073709551611, __src=0x7f4edc0f1dc7, __dest=<optimized out>) at /usr/include/x86_64-linux-gnu/bits/string_fortified.h:34

#23 _my_b_write (info=info@entry=0x7f4edc066490, Buffer=0x7f4edc0f1dc7 "", Count=Count@entry=7) at /data/src/10.2/mysys/mf_iocache.c:589

#24 0x0000560c0b59d5eb in my_b_write (Count=<optimized out>, Buffer=<optimized out>, info=0x7f4edc066490) at /data/src/10.2/include/my_sys.h:544

#25 merge_buffers (param=param@entry=0x7f4eee346410, from_file=from_file@entry=0x7f4edc0660c8, to_file=to_file@entry=0x7f4edc066490, sort_buffer=sort_buffer@entry=0x7f4edc0b1dc8 "", lastbuff=lastbuff@entry=0x7f4edc069088, Fb=Fb@entry=0x7f4edc069088, Tb=0x7f4edc0690b8, flag=1) at /data/src/10.2/sql/filesort.cc:1734

#26 0x0000560c0b59dd52 in merge_index (param=param@entry=0x7f4eee346410, sort_buffer=sort_buffer@entry=0x7f4edc0b1dc8 "", buffpek=buffpek@entry=0x7f4edc069088, maxbuffer=<optimized out>, tempfile=tempfile@entry=0x7f4edc0660c8, outfile=outfile@entry=0x7f4edc066490) at /data/src/10.2/sql/filesort.cc:1866

#27 0x0000560c0b4b9d90 in Unique::merge (this=this@entry=0x7f4edc066098, table=table@entry=0x7f4edc090a58, buff=buff@entry=0x7f4edc0b1dc8 "", without_last_merge=without_last_merge@entry=false) at /data/src/10.2/sql/uniques.cc:742

#28 0x0000560c0b4ba489 in Unique::get (this=this@entry=0x7f4edc066098, table=table@entry=0x7f4edc090a58) at /data/src/10.2/sql/uniques.cc:802

#29 0x0000560c0b6c6aef in read_keys_and_merge_scans (thd=0x7f4edc000c48, head=0x7f4edc090a58, quick_selects=..., pk_quick_select=0x0, read_record=0x7f4edc17a958, intersection=false, filtered_scans=0x0, unique_ptr=0x7f4edc17a8f0) at /data/src/10.2/sql/opt_range.cc:11105

#30 0x0000560c0b6c6da4 in QUICK_INDEX_MERGE_SELECT::read_keys_and_merge (this=0x7f4edc17a8b0) at /data/src/10.2/sql/sql_list.h:193

#31 0x0000560c0b43bd5c in join_init_read_record (tab=0x7f4edc014560) at /data/src/10.2/sql/sql_select.cc:19785

#32 0x0000560c0b427d6e in sub_select (end_of_records=false, join_tab=0x7f4edc014560, join=0x7f4edc011ab8) at /data/src/10.2/sql/sql_select.cc:18871

#33 sub_select (join=0x7f4edc011ab8, join_tab=0x7f4edc014560, end_of_records=<optimized out>) at /data/src/10.2/sql/sql_select.cc:18809

#34 0x0000560c0b450a07 in do_select (procedure=<optimized out>, join=0x7f4edc011ab8) at /data/src/10.2/sql/sql_select.cc:18418

#35 JOIN::exec_inner (this=this@entry=0x7f4edc011ab8) at /data/src/10.2/sql/sql_select.cc:3651

#36 0x0000560c0b450ca7 in JOIN::exec (this=0x7f4edc011ab8) at /data/src/10.2/sql/sql_select.cc:3446

#37 0x0000560c0b64a359 in subselect_single_select_engine::exec (this=0x7f4edc0117d8) at /data/src/10.2/sql/item_subselect.cc:4000

#38 0x0000560c0b649c8f in Item_subselect::exec (this=0x7f4edc0115d8) at /data/src/10.2/sql/item_subselect.cc:770

#39 0x0000560c0b650991 in Item_in_subselect::exec (this=0x7f4edc0115d8) at /data/src/10.2/sql/item_subselect.cc:950

#40 Item_in_subselect::val_bool (this=0x7f4edc0115d8) at /data/src/10.2/sql/item_subselect.cc:1855

#41 0x0000560c0b5da65d in Item_in_optimizer::val_int (this=0x7f4edc0120d0) at /data/src/10.2/sql/item_cmpfunc.cc:1677

#42 Item_in_optimizer::val_int (this=0x7f4edc0120d0) at /data/src/10.2/sql/item_cmpfunc.cc:1585

#43 0x0000560c0b5b0d25 in Item::val_bool (this=0x7f4edc0120d0) at /data/src/10.2/sql/item.cc:112

#44 0x0000560c0b5d09a3 in Item_func_not::val_int (this=0x7f4edc011818) at /data/src/10.2/sql/item_cmpfunc.cc:307

#45 0x0000560c0b49af8d in SQL_SELECT::skip_record (this=<optimized out>, thd=0x7f4edc000c48) at /data/src/10.2/sql/opt_range.h:1633

#46 mysql_update (thd=thd@entry=0x7f4edc000c48, table_list=<optimized out>, fields=..., values=..., conds=<optimized out>, order_num=<optimized out>, order=<optimized out>, limit=18446744073709551615, handle_duplicates=DUP_ERROR, ignore=false, found_return=0x7f4eee346f00, updated_return=0x7f4eee346fb0) at /data/src/10.2/sql/sql_update.cc:755

#47 0x0000560c0b3f0e60 in mysql_execute_command (thd=<optimized out>) at /data/src/10.2/sql/sql_parse.cc:4056

#48 0x0000560c0b3f724b in mysql_parse (thd=thd@entry=0x7f4edc000c48, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x7f4eee348520, is_com_multi=is_com_multi@entry=false, is_next_command=is_next_command@entry=false) at /data/src/10.2/sql/sql_parse.cc:7793

#49 0x0000560c0b3fa45d in dispatch_command (command=COM_QUERY, thd=0x7f4edc000c48, packet=<optimized out>, packet_length=<optimized out>, is_com_multi=<optimized out>, is_next_command=<optimized out>) at /data/src/10.2/sql/sql_class.h:1109

#50 0x0000560c0b3fb67d in do_command (thd=0x7f4edc000c48) at /data/src/10.2/sql/sql_parse.cc:1381

#51 0x0000560c0b4d4a06 in do_handle_one_connection (connect=connect@entry=0x560c0cfdeba8) at /data/src/10.2/sql/sql_connect.cc:1336

#52 0x0000560c0b4d4b7f in handle_one_connection (arg=arg@entry=0x560c0cfdeba8) at /data/src/10.2/sql/sql_connect.cc:1241

#53 0x0000560c0ba717a6 in pfs_spawn_thread (arg=0x560c0cfea2f8) at /data/src/10.2/storage/perfschema/pfs.cc:1869

#54 0x00007f4ef484d609 in start_thread (arg=<optimized out>) at pthread_create.c:477

#55 0x00007f4ef4444293 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

commit 658128af43b4d7c6db445164f8ed25ed4d1e3109

Author: Igor Babaev

Date:   Sun Feb 3 14:56:12 2019 -0800

    MDEV-16188 Use in-memory PK filters built from range index scans