Repeatable in maria-5.3 , maria-5.3-mwl89 . Not repeatable in maria-5.2, mysql-5.5

backtrace:

#4 <signal handler called>
#5 0x08333e4c in copy_fields (param=0xae533f60) at sql_select.cc:18231
#6 0x0832ccf2 in end_write (join=0xae52f2b8, join_tab=0xae52b708, end_of_records=false) at sql_select.cc:15166
#7 0x0832a73b in evaluate_join_record (join=0xae52f2b8, join_tab=0xae52b524, error=0) at sql_select.cc:14201
#8 0x0832a0ce in sub_select (join=0xae52f2b8, join_tab=0xae52b524, end_of_records=false) at sql_select.cc:14006
#9 0x083294a0 in do_select (join=0xae52f2b8, fields=0x0, table=0xae52d940, procedure=0x0) at sql_select.cc:13541
#10 0x0830e390 in JOIN::exec (this=0xae525980) at sql_select.cc:2091
#11 0x0824057c in subselect_single_select_engine::exec (this=0xae5142c8) at item_subselect.cc:2753
#12 0x0823ac0f in Item_subselect::exec (this=0xae5141c8) at item_subselect.cc:537
#13 0x0823ae0e in Item_in_subselect::exec (this=0xae5141c8) at item_subselect.cc:634
#14 0x0823c7f6 in Item_in_subselect::val_bool (this=0xae5141c8) at item_subselect.cc:1314
#15 0x081d9138 in Item::val_bool_result (this=0xae5141c8) at item.h:813
#16 0x082054d2 in Item_in_optimizer::val_int (this=0xae514548) at item_cmpfunc.cc:1980
#17 0x08329461 in do_select (join=0xae520ad8, fields=0xae525874, table=0x0, procedure=0x0) at sql_select.cc:13538
#18 0x0830fccb in JOIN::exec (this=0xae520ad8) at sql_select.cc:2530
#19 0x083104e8 in mysql_select (thd=0xb25bc08, rref_pointer_array=0xb25d6a4, tables=0xae513160, wild_num=0, fields=..., conds=0xae5141c8, og_num=1,
    order=0x0, group=0xae514360, having=0x0, proc_param=0x0, select_options=2147764736, result=0xae514408, unit=0xb25d304, select_lex=0xb25d5a0)
    at sql_select.cc:2749
#20 0x08308ac7 in handle_select (thd=0xb25bc08, lex=0xb25d2a8, result=0xae514408, setup_tables_done_option=0) at sql_select.cc:280
#21 0x082a5e15 in execute_sqlcom_select (thd=0xb25bc08, all_tables=0xae513160) at sql_parse.cc:5090
#22 0x0829cc87 in mysql_execute_command (thd=0xb25bc08) at sql_parse.cc:2235
#23 0x082a83b8 in mysql_parse (thd=0xb25bc08,
    rawbuf=0xae512ee8 "SELECT f11 FROM t2\nWHERE ( '0' ) IN (\nSELECT SQ3_alias1.f10 AS SQ3_field1 \nFROM t2 AS SQ3_alias1 JOIN t1 AS SQ3_alias2 ON SQ3_alias2.f3\nGROUP BY SQ3_field1\n)\nGROUP BY 1", length=168, found_semicolon=0xae883228) at sql_parse.cc:6090
#24 0x0829a910 in dispatch_command (command=COM_QUERY, thd=0xb25bc08, packet=0xb2b67d1 "", packet_length=168) at sql_parse.cc:1211
#25 0x08299dbd in do_command (thd=0xb25bc08) at sql_parse.cc:904
#26 0x08296e70 in handle_one_connection (arg=0xb25bc08) at sql_connect.cc:1154
#27 0x00116919 in start_thread () from /lib/libpthread.so.0
#28 0x0076acce in clone () from /lib/libc.so.6

explain:

id select_type table type possible_keys key key_len ref rows Extra
1 PRIMARY t2 ALL NULL NULL NULL NULL 2 Using temporary; Using filesort
2 DEPENDENT SUBQUERY SQ3_alias2 system NULL NULL NULL NULL 1 Using temporary; Using filesort
2 DEPENDENT SUBQUERY SQ3_alias1 ALL NULL NULL NULL NULL 2

test case:

CREATE TABLE t1 ( f3 int, f10 int) ;
INSERT IGNORE INTO t1 VALUES (4,0);

CREATE TABLE t2 ( f10 int, f11 int) ;
INSERT IGNORE INTO t2 VALUES (0,0),(0,0);

SELECT f11 FROM t2
WHERE ( '0' ) IN (
        SELECT SQ3_alias1.f10 AS SQ3_field1
        FROM t2 AS SQ3_alias1 JOIN t1 AS SQ3_alias2 ON SQ3_alias2.f3
        GROUP BY SQ3_field1
)
GROUP BY 1;