Server crash with SIGFPE in decimal2bin, ASAN global-buffer-overflow in decimal_bin_size, assertion `scale <= precision'

CREATE TABLE t (a DECIMAL(50,33), b INT);

INSERT INTO t VALUES (1.0,1),(2.0,2);

SELECT ROUND(a, b) AS f FROM t GROUP BY f WITH ROLLUP;

# Cleanup

DROP TABLE t;

200930  1:53:31 [ERROR] mysqld got signal 8 ;

#2  0x0000560eed60db18 in handle_fatal_signal (sig=8) at /data/src/10.2/sql/signal_handler.cc:343

#3  <signal handler called>

#4  0x0000560eedbb451a in decimal2bin (from=from@entry=0x7f09973ce330, to=to@entry=0x7f093c0243a9 "", precision=precision@entry=23, frac=frac@entry=33) at /data/src/10.2/strings/decimal.c:1275

#5  0x0000560eed71b350 in my_decimal2binary (mask=mask@entry=30, d=<optimized out>, bin=bin@entry=0x7f093c0243a9 "", prec=23, scale=33) at /data/src/10.2/sql/my_decimal.cc:215

#6  0x0000560eed60a66f in Type_handler_decimal_result::make_sort_key (this=<optimized out>, to=0x7f093c0243a9 "", item=0x7f093c00f5e0, sort_field=0x7f093c012588, param=<optimized out>) at /data/src/10.2/sql/filesort.cc:1150

#7  0x0000560eed60992b in Item::make_sort_key (param=0x7f09973ce590, sort_field=0x7f093c012588, item=0x7f093c00f5e0, to=0x7f093c0243a8 "\001", this=0x7f093c00f5e0) at /data/src/10.2/sql/item.h:933

#8  make_sortkey (param=0x7f09973ce590, to=0x7f093c0243a8 "\001", ref_pos=0x7f093c006608 "") at /data/src/10.2/sql/filesort.cc:1196

#9  0x0000560eed60cf31 in find_all_keys (found_rows=0x7f093c0176a0, pq=0x0, tempfile=0x7f09973ce620, buffpek_pointers=0x7f09973ce710, fs_info=0x7f093c017540, select=0x7f093c011b80, param=0x7f09973ce590, thd=0x7f093c000c48) at /data/src/10.2/sql/sql_array.h:62

#10 filesort (thd=0x7f093c000c48, table=table@entry=0x7f093c016548, filesort=filesort@entry=0x7f093c011e80, tracker=0x7f093c012510, join=join@entry=0x7f093c00ff78, first_table_bit=<optimized out>) at /data/src/10.2/sql/filesort.cc:275

#11 0x0000560eed4aa736 in create_sort_index (thd=<optimized out>, join=0x7f093c00ff78, tab=tab@entry=0x7f093c0112b0, fsort=0x7f093c011e80, fsort@entry=0x0) at /data/src/10.2/sql/sql_select.cc:22059

#12 0x0000560eed4aaa87 in st_join_table::sort_table (this=this@entry=0x7f093c0112b0) at /data/src/10.2/sql/sql_select.cc:19821

#13 0x0000560eed4aaaed in join_init_read_record (tab=0x7f093c0112b0) at /data/src/10.2/sql/sql_select.cc:19762

#14 0x0000560eed496b0e in sub_select (end_of_records=false, join_tab=0x7f093c0112b0, join=0x7f093c00ff78) at /data/src/10.2/sql/sql_select.cc:18853

#15 sub_select (join=0x7f093c00ff78, join_tab=0x7f093c0112b0, end_of_records=<optimized out>) at /data/src/10.2/sql/sql_select.cc:18791

#16 0x0000560eed4bf757 in do_select (procedure=<optimized out>, join=0x7f093c00ff78) at /data/src/10.2/sql/sql_select.cc:18400

#17 JOIN::exec_inner (this=this@entry=0x7f093c00ff78) at /data/src/10.2/sql/sql_select.cc:3638

#18 0x0000560eed4bf9f7 in JOIN::exec (this=this@entry=0x7f093c00ff78) at /data/src/10.2/sql/sql_select.cc:3433

#19 0x0000560eed4bfb3a in mysql_select (thd=0x7f093c000c48, tables=0x7f093c00f710, wild_num=0, fields=..., conds=0x0, og_num=1, order=0x0, group=0x7f093c00fe48, having=0x0, proc_param=0x0, select_options=2147748608, result=0x7f093c00ff58, unit=0x7f093c004680, select_lex=0x7f093c004dc0) at /data/src/10.2/sql/sql_select.cc:3833

#20 0x0000560eed4c04af in handle_select (thd=thd@entry=0x7f093c000c48, lex=lex@entry=0x7f093c0045c0, result=result@entry=0x7f093c00ff58, setup_tables_done_option=setup_tables_done_option@entry=0) at /data/src/10.2/sql/sql_select.cc:361

#21 0x0000560eed455b91 in execute_sqlcom_select (thd=0x7f093c000c48, all_tables=0x7f093c00f710) at /data/src/10.2/sql/sql_parse.cc:6218

#22 0x0000560eed4631f2 in mysql_execute_command (thd=0x7f093c000c48) at /data/src/10.2/sql/sql_parse.cc:3524

#23 0x0000560eed4660eb in mysql_parse (thd=thd@entry=0x7f093c000c48, rawbuf=<optimized out>, length=<optimized out>, parser_state=parser_state@entry=0x7f09973d05e0, is_com_multi=is_com_multi@entry=false, is_next_command=is_next_command@entry=false) at /data/src/10.2/sql/sql_parse.cc:7733

#24 0x0000560eed46939d in dispatch_command (command=COM_QUERY, thd=0x7f093c000c48, packet=<optimized out>, packet_length=<optimized out>, is_com_multi=<optimized out>, is_next_command=<optimized out>) at /data/src/10.2/sql/sql_class.h:1095

#25 0x0000560eed46a527 in do_command (thd=0x7f093c000c48) at /data/src/10.2/sql/sql_parse.cc:1377

#26 0x0000560eed542876 in do_handle_one_connection (connect=connect@entry=0x560eef9af7d8) at /data/src/10.2/sql/sql_connect.cc:1336

#27 0x0000560eed5429ef in handle_one_connection (arg=0x560eef9af7d8) at /data/src/10.2/sql/sql_connect.cc:1241

#28 0x00007f09aeef1609 in start_thread (arg=<optimized out>) at pthread_create.c:477

#29 0x00007f09aeae6293 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

==2939127==ERROR: AddressSanitizer: global-buffer-overflow on address 0x563bd47ec09c at pc 0x563bd424b204 bp 0x7f51cdf87180 sp 0x7f51cdf87170

READ of size 4 at 0x563bd47ec09c thread T5

    #0 0x563bd424b203 in decimal_bin_size /data/src/10.2/strings/decimal.c:1486

    #1 0x563bd2b8ffea in my_decimal_get_binary_size(unsigned int, unsigned int) /data/src/10.2/sql/my_decimal.h:263

    #2 0x563bd2b8ffea in calc_group_buffer /data/src/10.2/sql/sql_select.cc:23161

    #3 0x563bd2c4d7f6 in JOIN::optimize_inner() /data/src/10.2/sql/sql_select.cc:1976

    #4 0x563bd2c534f1 in JOIN::optimize() /data/src/10.2/sql/sql_select.cc:1114

    #5 0x563bd2c534f1 in JOIN::optimize() /data/src/10.2/sql/sql_select.cc:1106

    #6 0x563bd2c5b1c8 in mysql_select(THD*, TABLE_LIST*, unsigned int, List<Item>&, Item*, unsigned int, st_order*, st_order*, Item*, st_order*, unsigned long long, select_result*, st_select_lex_unit*, st_select_lex*) /data/src/10.2/sql/sql_select.cc:3819

    #7 0x563bd2c5bec4 in handle_select(THD*, LEX*, select_result*, unsigned long) /data/src/10.2/sql/sql_select.cc:361

    #8 0x563bd2af5ea4 in execute_sqlcom_select /data/src/10.2/sql/sql_parse.cc:6218

    #9 0x563bd2b2184f in mysql_execute_command(THD*) /data/src/10.2/sql/sql_parse.cc:3524

    #10 0x563bd2b2aee7 in mysql_parse(THD*, char*, unsigned int, Parser_state*, bool, bool) /data/src/10.2/sql/sql_parse.cc:7733

    #11 0x563bd2b3484e in dispatch_command(enum_server_command, THD*, char*, unsigned int, bool, bool) /data/src/10.2/sql/sql_parse.cc:1823

    #12 0x563bd2b39004 in do_command(THD*) /data/src/10.2/sql/sql_parse.cc:1377

    #13 0x563bd2e1fd2a in do_handle_one_connection(CONNECT*) /data/src/10.2/sql/sql_connect.cc:1336

    #14 0x563bd2e20400 in handle_one_connection /data/src/10.2/sql/sql_connect.cc:1241

    #15 0x563bd404a2f7 in pfs_spawn_thread /data/src/10.2/storage/perfschema/pfs.cc:1869

    #16 0x7f51d8bd8608 in start_thread /build/glibc-ZN95T4/glibc-2.31/nptl/pthread_create.c:477

    #17 0x7f51d87b4292 in __clone (/lib/x86_64-linux-gnu/libc.so.6+0x122292)

0x563bd47ec09c is located 4 bytes to the left of global variable 'dig2bytes' defined in '/data/src/10.2/strings/decimal.c:132:18' (0x563bd47ec0a0) of size 40

0x563bd47ec09c is located 28 bytes to the right of global variable 'frac_max' defined in '/data/src/10.2/strings/decimal.c:133:19' (0x563bd47ec060) of size 32

SUMMARY: AddressSanitizer: global-buffer-overflow /data/src/10.2/strings/decimal.c:1486 in decimal_bin_size

Shadow bytes around the buggy address:

  0x0ac7fa8f57c0: 00 00 05 f9 f9 f9 f9 f9 00 00 00 00 00 00 00 00

  0x0ac7fa8f57d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00

  0x0ac7fa8f57e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00

  0x0ac7fa8f57f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00

  0x0ac7fa8f5800: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00

=>0x0ac7fa8f5810: f9 f9 f9[f9]00 00 00 00 00 f9 f9 f9 f9 f9 f9 f9

  0x0ac7fa8f5820: 00 00 00 00 00 f9 f9 f9 f9 f9 f9 f9 00 00 00 00

  0x0ac7fa8f5830: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00

  0x0ac7fa8f5840: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00

  0x0ac7fa8f5850: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00

  0x0ac7fa8f5860: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00

Shadow byte legend (one shadow byte represents 8 application bytes):

  Addressable:           00

  Partially addressable: 01 02 03 04 05 06 07 

  Heap left redzone:       fa

  Freed heap region:       fd

  Stack left redzone:      f1

  Stack mid redzone:       f2

  Stack right redzone:     f3

  Stack after return:      f5

  Stack use after scope:   f8

  Global redzone:          f9

  Global init order:       f6

  Poisoned by user:        f7

  Container overflow:      fc

  Array cookie:            ac

  Intra object redzone:    bb

  ASan internal:           fe

  Left alloca redzone:     ca

  Right alloca redzone:    cb

  Shadow gap:              cc

Thread T5 created by T0 here:

    #0 0x7f51d8d03805 in pthread_create (/lib/x86_64-linux-gnu/libasan.so.5+0x3a805)

    #1 0x563bd4052f57 in spawn_thread_v1 /data/src/10.2/storage/perfschema/pfs.cc:1919

==2939127==ABORTING

mysqld: /data/src/10.2/strings/decimal.c:1485: decimal_bin_size: Assertion `scale <= precision' failed.

200930  1:58:30 [ERROR] mysqld got signal 6 ;

#7  0x00007f6b9646ef36 in __GI___assert_fail (assertion=0x560b0baac253 "scale <= precision", file=0x560b0baabfc0 "/data/src/10.2/strings/decimal.c", line=1485, function=0x560b0baac4b0 <__PRETTY_FUNCTION__.14205> "decimal_bin_size") at assert.c:101

#8  0x0000560b0b678d65 in decimal_bin_size (precision=23, scale=33) at /data/src/10.2/strings/decimal.c:1485

#9  0x0000560b0ac852b5 in my_decimal_get_binary_size (precision=23, scale=33) at /data/src/10.2/sql/my_decimal.h:263

#10 0x0000560b0ac7a382 in calc_group_buffer (join=0x7f6b800133d0, group=0x7f6b80013290) at /data/src/10.2/sql/sql_select.cc:23161

#11 0x0000560b0ac42dc0 in JOIN::optimize_inner (this=0x7f6b800133d0) at /data/src/10.2/sql/sql_select.cc:1976

#12 0x0000560b0ac3fba6 in JOIN::optimize (this=0x7f6b800133d0) at /data/src/10.2/sql/sql_select.cc:1114

#13 0x0000560b0ac490e4 in mysql_select (thd=0x7f6b80000d90, tables=0x7f6b80012b58, wild_num=0, fields=..., conds=0x0, og_num=1, order=0x0, group=0x7f6b80013290, having=0x0, proc_param=0x0, select_options=2147748608, result=0x7f6b800133b0, unit=0x7f6b80004988, select_lex=0x7f6b800050c8) at /data/src/10.2/sql/sql_select.cc:3819

#14 0x0000560b0ac3d31a in handle_select (thd=0x7f6b80000d90, lex=0x7f6b800048c8, result=0x7f6b800133b0, setup_tables_done_option=0) at /data/src/10.2/sql/sql_select.cc:361

#15 0x0000560b0ac07eb8 in execute_sqlcom_select (thd=0x7f6b80000d90, all_tables=0x7f6b80012b58) at /data/src/10.2/sql/sql_parse.cc:6218

#16 0x0000560b0abfe77a in mysql_execute_command (thd=0x7f6b80000d90) at /data/src/10.2/sql/sql_parse.cc:3524

#17 0x0000560b0ac0bc47 in mysql_parse (thd=0x7f6b80000d90, rawbuf=0x7f6b800126f8 "SELECT ROUND(a, b) AS f FROM t GROUP BY f WITH ROLLUP", length=53, parser_state=0x7f6b905c95f0, is_com_multi=false, is_next_command=false) at /data/src/10.2/sql/sql_parse.cc:7733

#18 0x0000560b0abf9f56 in dispatch_command (command=COM_QUERY, thd=0x7f6b80000d90, packet=0x7f6b80008b51 "SELECT ROUND(a, b) AS f FROM t GROUP BY f WITH ROLLUP", packet_length=53, is_com_multi=false, is_next_command=false) at /data/src/10.2/sql/sql_parse.cc:1823

#19 0x0000560b0abf8a51 in do_command (thd=0x7f6b80000d90) at /data/src/10.2/sql/sql_parse.cc:1377

#20 0x0000560b0ad52695 in do_handle_one_connection (connect=0x560b0dc8faa0) at /data/src/10.2/sql/sql_connect.cc:1336

#21 0x0000560b0ad523fa in handle_one_connection (arg=0x560b0dc8faa0) at /data/src/10.2/sql/sql_connect.cc:1241

#22 0x0000560b0b576a6c in pfs_spawn_thread (arg=0x560b0dc98770) at /data/src/10.2/storage/perfschema/pfs.cc:1869

#23 0x00007f6b96980609 in start_thread (arg=<optimized out>) at pthread_create.c:477

#24 0x00007f6b9655a293 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95