diff --git a/mysql-test/main/mysqld--help.result b/mysql-test/main/mysqld--help.result index 79c0d7005f2..f097d0e478f 100644 --- a/mysql-test/main/mysqld--help.result +++ b/mysql-test/main/mysqld--help.result @@ -723,6 +723,14 @@ The following specify which files/extra groups are read (specified before remain in MariaDB 11.0 as it is not needed with the new 11.0 optimizer. Use 'ALL' to set all combinations. + --optimizer-join-limit-pref-ratio=# + For queries with JOIN and ORDER BY LIMIT : change the + join plan to one that can short-cut after producing + #LIMIT matches if that promises N times speedup. (That + is, a conservative setting is a high value, like var=100 + to change only if this promises 100x) The default is 0 + which gives old behavior (don't change no matter what the + speedup) --optimizer-max-sel-arg-weight=# The maximum weight of the SEL_ARG graph. Set to 0 for no limit @@ -1696,6 +1704,7 @@ old-mode UTF8_IS_UTF8MB3 old-passwords FALSE old-style-user-limits FALSE optimizer-adjust-secondary-key-costs +optimizer-join-limit-pref-ratio 0 optimizer-max-sel-arg-weight 32000 optimizer-max-sel-args 16000 optimizer-prune-level 1 diff --git a/sql/sql_class.h b/sql/sql_class.h index d49f550cd1a..797f78eb70c 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -758,6 +758,7 @@ typedef struct system_variables ulong net_retry_count; ulong net_wait_timeout; ulong net_write_timeout; + ulong optimizer_join_limit_pref_ratio; ulong optimizer_prune_level; ulong optimizer_search_depth; ulong optimizer_selectivity_sampling_limit; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 7aed2432e03..64f45755f50 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -330,6 +330,7 @@ static void optimize_rownum(THD *thd, SELECT_LEX_UNIT *unit, Item *cond); static bool process_direct_rownum_comparison(THD *thd, SELECT_LEX_UNIT *unit, Item *cond); +bool join_shortcut_limit_is_applicable(JOIN *join); #ifndef DBUG_OFF /* @@ -5795,6 +5796,13 @@ make_join_statistics(JOIN *join, List &tables_list, join->sort_by_table= get_sort_by_table(join->order, join->group_list, join->select_lex->leaf_tables, join->const_table_map); + + join->limit_shortcut_applicable= join_shortcut_limit_is_applicable(join); + /* + psergey-todo: check the applicability and figure which indexes we can use. + TODO: move this to after the join optimization? + */ + /* Update info on indexes that can be used for search lookups as reading const tables may has added new sargable predicates. @@ -10322,6 +10330,139 @@ check_if_edge_table(POSITION *pos, } +/* + Check if we would be able to short-cut join execution for ORDER BY ... LIMIT +*/ +bool join_shortcut_limit_is_applicable(JOIN *join) +{ + /* + Any post-join operation like GROUP BY or DISTINCT or window functions + means we cannot short-cut join execution + */ + if (!join->thd->variables.optimizer_join_limit_pref_ratio || + !join->order || + join->select_limit == HA_POS_ERROR || + join->group_list || + join->select_distinct || + join->select_options & SELECT_BIG_RESULT || + join->rollup.state != ROLLUP::STATE_NONE || + join->select_lex->have_window_funcs() + // || TODO: aggregates and implicit grouping + ) + { + return false; + } + + /* If sorting is not done by one table can't do that either */ + if (!join->sort_by_table) + return false; + + /* It looks like we can short-cut limit due to join */ + return true; +} + + +JOIN_TAB **join_check_shortcut_limit_now(JOIN *join, uint idx) +{ + if (join->limit_shortcut_applicable && idx == join->const_tables && + !join->emb_sjm_nest && + join->join_record_count > join->select_limit && + join->best_positions[join->const_tables].table->table != join->sort_by_table) + { + JOIN_TAB **sort_tbl; + for (sort_tbl= join->best_ref + idx ; *sort_tbl ; sort_tbl++) + { + if ((*sort_tbl)->table == join->sort_by_table) + return sort_tbl; + } + } + return NULL; +} + +class Shortcut_opt +{ +public: + double save_join_record_count; + double save_best_read; + POSITION *save_best_pos; +}; + + +Shortcut_opt *join_start_shortcut_limit_run(JOIN *join) +{ + // We expect that there is some query plan already. + DBUG_ASSERT(join->best_read < DBL_MAX); + + Shortcut_opt *opt; + POSITION *pos; + if (!multi_alloc_root(join->thd->mem_root, + &opt, sizeof(Shortcut_opt), + &pos, + sizeof(POSITION)*(join->table_count + 1), + NullS)) + return NULL; + + memcpy(pos, join->best_positions, sizeof(POSITION)*join->table_count); + opt->save_join_record_count= join->join_record_count; + opt->save_best_read= join->best_read; + opt->save_best_pos= pos; + + join->best_read= DBL_MAX; + join->limit_optimization_mode=1; + return opt; +} + +void join_end_shortcut_limit_run(JOIN *join, Shortcut_opt *opt) +{ + join->limit_optimization_mode= false; + + if (join->best_read < DBL_MAX) + { + /* We have produced a query plan with a matching join order */ + double fraction= 1.0; + if (join->join_record_count > join->select_limit) + { + fraction= join->select_limit / join->join_record_count; + } + + /* + TODO: here, check if the first table's access method produces the + required ordering. + Possible options: + 1. Yes: we can just take a fraction of the execution cost. + 2A No: change the access method to one that does produce + the required ordering, update the costs. + 2B No: Need to pass the first table to filesort(). + */ + + // This is 1: + double limited_cost= join->best_read * fraction; + double needed_speedup = join->thd->variables.optimizer_join_limit_pref_ratio; + if (limited_cost * needed_speedup < opt->save_best_read) + { + // LIMIT plan is cheaper. + // It is already in join->best_positions so do nothing + // (TODO: update the costs) + // (TODO: switch to using a suitable index?) + return; + } + } + + // Restore back the original plan + memcpy(join->best_positions, opt->save_best_pos, + sizeof(POSITION)*join->table_count); + join->join_record_count= opt->save_join_record_count; + join->best_read= opt->save_best_read; +/* + bool fatal_err; + test_if_skip_sort_order(tab, join->order, join->select_limit, + true, // no_changes + &tab->table->keys_in_use_for_order_by, + &fatal_err); +*/ +} + + /** Find a good, possibly optimal, query execution plan (QEP) by a possibly exhaustive search. @@ -10489,6 +10630,8 @@ best_extension_by_limited_search(JOIN *join, if (join->emb_sjm_nest) allowed_tables= join->emb_sjm_nest->sj_inner_tables & ~join->const_table_map; + Shortcut_opt *optimizing_shortcut= NULL; + for (pos= join->best_ref + idx ; (s= *pos) ; pos++) { table_map real_table_bit= s->table->map; @@ -10677,6 +10820,21 @@ best_extension_by_limited_search(JOIN *join, goto end; } } + + if (optimizing_shortcut) + { + join_end_shortcut_limit_run(join, optimizing_shortcut); + break; + } + + JOIN_TAB **sort_tbl; + if ((sort_tbl= join_check_shortcut_limit_now(join, idx))) + { + // Do another pass by putting the table of interest first. + if (!(optimizing_shortcut= join_start_shortcut_limit_run(join))) + DBUG_RETURN(SEARCH_ERROR); + pos= sort_tbl - 1; + } } best_res= SEARCH_OK; diff --git a/sql/sql_select.h b/sql/sql_select.h index 37a906b50aa..9e7d6e4791a 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -1205,6 +1205,10 @@ class JOIN :public Sql_alloc passing 1st non-const table to filesort(). NULL means no such table exists. */ TABLE *sort_by_table; + + bool limit_shortcut_applicable; + bool limit_optimization_mode; + /* Number of tables in the join. (In MySQL, it is named 'tables' and is also the number of elements in diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index f6fe8847d08..bd2e614f8eb 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -2702,6 +2702,20 @@ static Sys_var_ulong Sys_optimizer_selectivity_sampling_limit( VALID_RANGE(SELECTIVITY_SAMPLING_THRESHOLD, UINT_MAX), DEFAULT(SELECTIVITY_SAMPLING_LIMIT), BLOCK_SIZE(1)); +static Sys_var_ulong Sys_optimizer_join_limit_pref_ratio( + "optimizer_join_limit_pref_ratio", + "For queries with JOIN and ORDER BY LIMIT : change the join plan " + "to one that can short-cut after producing #LIMIT matches if that " + "promises N times speedup. " + "(That is, a conservative setting is a high value, like var=100 to " + "change only if this promises 100x) " + "The default is 0 which gives old behavior (don't change no matter " + "what the speedup)", + SESSION_VAR(optimizer_join_limit_pref_ratio), + CMD_LINE(REQUIRED_ARG), + VALID_RANGE(0, UINT_MAX), + DEFAULT(0), BLOCK_SIZE(1)); + static Sys_var_ulong Sys_optimizer_use_condition_selectivity( "optimizer_use_condition_selectivity", "Controls selectivity of which conditions the optimizer takes into "