Run these queries in debug build:

CREATE TABLE t0 ( c11 VARBINARY ( 241 ) , INDEX i0 ( c11 ( 23 ) ) , c12 TEXT , c29 DOUBLE ( 71 , 7 ) SIGNED DEFAULT ( -29 ) ) ;
INSERT INTO t0 VALUES ( 115 , -87 , DEFAULT ) , ( 29 , 44 , DEFAULT ) ;
CREATE UNIQUE INDEX i1 ON t0 ( c11 ) ;
INSERT INTO t0 VALUES ( -7446687632669362065 , 107 , DEFAULT ) , ( 1 , -82 , DEFAULT ) ;
SELECT ROUND ( -108 , -50 ) SOUNDS LIKE TRIM( TRAILING FROM -30 ) AND ROW_NUMBER ( ) OVER ( PARTITION BY t0 . c11 , t0 . c11 , t0 . c11 , BIT_AND( EXPORT_SET ( t0 . c11 , 'y\'XAcl0L<[$~pCXymGs`eA:yc)=d+oMBQNWg<6b}4<?,54e5~A#' , LOCATE ( t0 . c11 , '7p*@p. OZ#2KFc$B5fhkls6/S=#]@#tot

:ve_XZ{' , '[2z~uG!UNxmx2`Dh5p
dD' ) / EXP ( t0 . c29 ) = t0 . c11 IS NOT NULL , 'dsevBo [=2D<Fi' ) SOUNDS LIKE TRUNCATE ( -21 , 49 ) = t0 . c12 ) OVER ( ) ) OR LOG ( 125 , 40 ) > t0 . c11 AS c51 , ( SELECT c11 AS c6 FROM t0 LIMIT 1 ) IN ( -120 , ROUND ( -33 , -6774865926151429870 ) SOUNDS LIKE TRIM( TRAILING FROM -50 ) AND ROW_NUMBER ( ) OVER ( PARTITION BY -76 ) ) AS c50 , -40 AS c53 FROM t0 WHERE TAN ( 61 ) BETWEEN -112 AND 7 IS NOT NULL GROUP BY c11 , c12 , c29 , c11 , c29 WITH ROLLUP HAVING RAND ( ) ;

Will trigger Assertion Failed.
GDB info:
#0 0x00007ffff6c0c00b in raise () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007ffff6beb859 in abort () from /lib/x86_64-linux-gnu/libc.so.6
#2 0x00007ffff6beb729 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#3 0x00007ffff6bfcfd6 in __assert_fail () from /lib/x86_64-linux-gnu/libc.so.6
#4 0x0000555557bf5bc9 in Item_func_eq::val_int (this=0x6290000f8820) at /home/wx/mariadb-11.3.0/sql/item_cmpfunc.cc:1779
#5 0x0000555557e034e6 in Item_sum_and::add (this=0x6290000f8a58) at /home/wx/mariadb-11.3.0/sql/item_sum.cc:2747
#6 0x00005555579135a9 in Frame_cursor::add_value_to_items (this=0x62d00003d758) at /home/wx/mariadb-11.3.0/sql/sql_window.cc:1166
#7 0x0000555557915cdd in Frame_range_current_row_bottom::pre_next_partition (this=0x62d00003d758, rownum=0) at /home/wx/mariadb-11.3.0/sql/sql_window.cc:1576
#8 0x0000555557913cfb in Cursor_manager::notify_cursors_partition_changed (this=0x603000232690, rownum=0) at /home/wx/mariadb-11.3.0/sql/sql_window.cc:1225
#9 0x000055555790ed44 in compute_window_func (thd=0x62c0001e0288, window_functions=..., cursor_managers=..., tbl=0x6240006fa1a8, filesort_result=0x6150001c5280) at /home/wx/mariadb-11.3.0/sql/sql_window.cc:2907
#10 0x000055555790f54d in Window_func_runner::exec (this=0x6290001fcf38, thd=0x62c0001e0288, tbl=0x6240006fa1a8, filesort_result=0x6150001c5280) at /home/wx/mariadb-11.3.0/sql/sql_window.cc:3039
#11 0x000055555790f79c in Window_funcs_sort::exec (this=0x6290001fcf30, join=0x6290001f02c0, keep_filesort_result=true) at /home/wx/mariadb-11.3.0/sql/sql_window.cc:3067
#12 0x00005555579105c8 in Window_funcs_computation::exec (this=0x6290001fcf08, join=0x6290001f02c0, keep_last_filesort_result=true) at /home/wx/mariadb-11.3.0/sql/sql_window.cc:3196
#13 0x00005555573a1ea2 in AGGR_OP::end_send (this=0x6290001fc418) at /home/wx/mariadb-11.3.0/sql/sql_select.cc:32300
#14 0x000055555735ebd8 in sub_select_postjoin_aggr (join=0x6290001f02c0, join_tab=0x6290001f78f0, end_of_records=true) at /home/wx/mariadb-11.3.0/sql/sql_select.cc:23128
#15 0x000055555735f713 in sub_select (join=0x6290001f02c0, join_tab=0x6290001f7478, end_of_records=true) at /home/wx/mariadb-11.3.0/sql/sql_select.cc:23378
#16 0x000055555735dbe3 in do_select (join=0x6290001f02c0, procedure=0x0) at /home/wx/mariadb-11.3.0/sql/sql_select.cc:22963
#17 0x00005555572dbfe9 in JOIN::exec_inner (this=0x6290001f02c0) at /home/wx/mariadb-11.3.0/sql/sql_select.cc:4941
#18 0x00005555572d93a0 in JOIN::exec (this=0x6290001f02c0) at /home/wx/mariadb-11.3.0/sql/sql_select.cc:4718
#19 0x00005555572ddbab in mysql_select (thd=0x62c0001e0288, tables=0x6290001ed3d8, fields=..., conds=0x6290001edf58, og_num=8, order=0x0, group=0x6290000f6890, having=0x6290001ef880, proc_param=0x0, select_options=2165049856, result=0x6290001f0290, unit=0x62c0001e46d8, select_lex=0x6290000f5950) at /home/wx/mariadb-11.3.0/sql/sql_select.cc:5249
#20 0x00005555572ad18a in handle_select (thd=0x62c0001e0288, lex=0x62c0001e45f8, result=0x6290001f0290, setup_tables_done_option=0) at /home/wx/mariadb-11.3.0/sql/sql_select.cc:628
#21 0x00005555571ce583 in execute_sqlcom_select (thd=0x62c0001e0288, all_tables=0x6290001ed3d8) at /home/wx/mariadb-11.3.0/sql/sql_parse.cc:6013
#22 0x00005555571becf6 in mysql_execute_command (thd=0x62c0001e0288, is_called_from_prepared_stmt=false) at /home/wx/mariadb-11.3.0/sql/sql_parse.cc:3912
#23 0x00005555571d95e2 in mysql_parse (thd=0x62c0001e0288, rawbuf=0x6290000f52a8 "SELECT ROUND ( -108 , -50 ) SOUNDS LIKE TRIM( TRAILING FROM -30 ) AND ROW_NUMBER ( ) OVER ( PARTITION BY t0 . c11 , t0 . c11 , t0 . c11 , BIT_AND( EXPORT_SET ( t0 . c11 , 'y
'XAcl0L<[$~pCXymGs`eA:yc)="..., length=798, parser_state=0x7fffe33b0870) at /home/wx/mariadb-11.3.0/sql/sql_parse.cc:7734
#24 0x00005555571b1237 in dispatch_command (command=COM_QUERY, thd=0x62c0001e0288, packet=0x6290000fa289 "", packet_length=801, blocking=true) at /home/wx/mariadb-11.3.0/sql/sql_parse.cc:1893
#25 0x00005555571adf7c in do_command (thd=0x62c0001e0288, blocking=true) at /home/wx/mariadb-11.3.0/sql/sql_parse.cc:1406
#26 0x000055555768e557 in do_handle_one_connection (connect=0x61100008d608, put_in_cache=true) at /home/wx/mariadb-11.3.0/sql/sql_connect.cc:1445
#27 0x000055555768deb4 in handle_one_connection (arg=0x61100008d4c8) at /home/wx/mariadb-11.3.0/sql/sql_connect.cc:1347
#28 0x00005555582fa350 in pfs_spawn_thread (arg=0x618000005108) at /home/wx/mariadb-11.3.0/storage/perfschema/pfs.cc:2201
#29 0x00007ffff7115609 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#30 0x00007ffff6ce8133 in clone () from /lib/x86_64-linux-gnu/libc.so.6