diff --git a/mysql-test/r/all_persisted_variables.result b/mysql-test/r/all_persisted_variables.result index afe655d4555e..79dac7d7a610 100644 --- a/mysql-test/r/all_persisted_variables.result +++ b/mysql-test/r/all_persisted_variables.result @@ -44,7 +44,7 @@ include/assert.inc ['Expect 500+ variables in the table. Due to open Bugs, we ar # Test SET PERSIST -include/assert.inc ['Expect 398 persisted variables in the table. Due to open Bugs, we are checking for 392'] +include/assert.inc ['Expect 399 persisted variables in the table. Due to open Bugs, we are checking for 393'] ************************************************************ * 3. Restart server, it must preserve the persisted variable @@ -52,9 +52,9 @@ include/assert.inc ['Expect 398 persisted variables in the table. Due to open Bu ************************************************************ # restart -include/assert.inc ['Expect 392 persisted variables in persisted_variables table.'] -include/assert.inc ['Expect 392 persisted variables shown as PERSISTED in variables_info table.'] -include/assert.inc ['Expect 392 persisted variables with matching peristed and global values.'] +include/assert.inc ['Expect 393 persisted variables in persisted_variables table.'] +include/assert.inc ['Expect 393 persisted variables shown as PERSISTED in variables_info table.'] +include/assert.inc ['Expect 393 persisted variables with matching peristed and global values.'] ************************************************************ * 4. Test RESET PERSIST IF EXISTS. Verify persisted variable diff --git a/mysql-test/r/flush_only_old_table_cache_entries.result b/mysql-test/r/flush_only_old_table_cache_entries.result new file mode 100644 index 000000000000..ccf4e922e69f --- /dev/null +++ b/mysql-test/r/flush_only_old_table_cache_entries.result @@ -0,0 +1,7 @@ +SET @start_global_value = @@global.flush_only_old_table_cache_entries; +SET @@global.debug= '+d,tdc_flush_unused_tables'; +SET debug_sync = 'now WAIT_FOR signal.disabled'; +SET @@global.flush_only_old_table_cache_entries = true; +SET debug_sync = 'now WAIT_FOR signal.enabled'; +SET @@global.debug= '-d,tdc_flush_unused_tables'; +SET @@global.flush_only_old_table_cache_entries = @start_global_value; diff --git a/mysql-test/r/mysqld--help-notwin.result b/mysql-test/r/mysqld--help-notwin.result index 2c2a8aa89afe..fc32511c13a8 100644 --- a/mysql-test/r/mysqld--help-notwin.result +++ b/mysql-test/r/mysqld--help-notwin.result @@ -330,6 +330,9 @@ The following options may be given as the first argument: The max size of a file to use for filesort. Raise an error when this is exceeded. 0 means no limit. --flush Flush MyISAM tables to disk between SQL commands + --flush-only-old-table-cache-entries + Enable/disable flushing table and definition cache + entries policy based on TTL specified by flush_time. --flush-time=# A dedicated thread is created to flush all tables at the given interval --ft-boolean-syntax=name @@ -1501,6 +1504,7 @@ explicit-defaults-for-timestamp TRUE external-locking FALSE filesort-max-file-size 0 flush FALSE +flush-only-old-table-cache-entries FALSE flush-time 0 ft-boolean-syntax + -><()~*:""&| ft-max-word-len 84 diff --git a/mysql-test/suite/sys_vars/r/flush_only_old_table_cache_entries_basic.result b/mysql-test/suite/sys_vars/r/flush_only_old_table_cache_entries_basic.result new file mode 100644 index 000000000000..76709ee34309 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/flush_only_old_table_cache_entries_basic.result @@ -0,0 +1,67 @@ +set @start_global_value = @@global.flush_only_old_table_cache_entries; +select @@global.flush_only_old_table_cache_entries; +@@global.flush_only_old_table_cache_entries +0 +select @@session.flush_only_old_table_cache_entries; +ERROR HY000: Variable 'flush_only_old_table_cache_entries' is a GLOBAL variable +show global variables like 'flush_only_old_table_cache_entries'; +Variable_name Value +flush_only_old_table_cache_entries OFF +show session variables like 'flush_only_old_table_cache_entries'; +Variable_name Value +flush_only_old_table_cache_entries OFF +select * from performance_schema.global_variables where variable_name='flush_only_old_table_cache_entries'; +VARIABLE_NAME VARIABLE_VALUE +flush_only_old_table_cache_entries OFF +select * from performance_schema.session_variables where variable_name='flush_only_old_table_cache_entries'; +VARIABLE_NAME VARIABLE_VALUE +flush_only_old_table_cache_entries OFF +set global flush_only_old_table_cache_entries = ON; +select @@global.flush_only_old_table_cache_entries; +@@global.flush_only_old_table_cache_entries +1 +set session flush_only_old_table_cache_entries = ON; +ERROR HY000: Variable 'flush_only_old_table_cache_entries' is a GLOBAL variable and should be set with SET GLOBAL +set @@global.flush_only_old_table_cache_entries = ON; +select @@global.flush_only_old_table_cache_entries; +@@global.flush_only_old_table_cache_entries +1 +set @@global.flush_only_old_table_cache_entries = OFF; +select @@global.flush_only_old_table_cache_entries; +@@global.flush_only_old_table_cache_entries +0 +set @@global.flush_only_old_table_cache_entries = 0; +select @@global.flush_only_old_table_cache_entries; +@@global.flush_only_old_table_cache_entries +0 +set @@global.flush_only_old_table_cache_entries = 1; +select @@global.flush_only_old_table_cache_entries; +@@global.flush_only_old_table_cache_entries +1 +set @@global.flush_only_old_table_cache_entries = TRUE; +select @@global.flush_only_old_table_cache_entries; +@@global.flush_only_old_table_cache_entries +1 +set @@global.flush_only_old_table_cache_entries = FALSE; +select @@global.flush_only_old_table_cache_entries; +@@global.flush_only_old_table_cache_entries +0 +set @@global.flush_only_old_table_cache_entries = 'ONN'; +ERROR 42000: Variable 'flush_only_old_table_cache_entries' can't be set to the value of 'ONN' +set @@global.flush_only_old_table_cache_entries = "OFFF"; +ERROR 42000: Variable 'flush_only_old_table_cache_entries' can't be set to the value of 'OFFF' +set @@global.flush_only_old_table_cache_entries = OF; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'OF' at line 1 +set @@global.flush_only_old_table_cache_entries = TTRUE; +ERROR 42000: Variable 'flush_only_old_table_cache_entries' can't be set to the value of 'TTRUE' +set @@global.flush_only_old_table_cache_entries = FELSE; +ERROR 42000: Variable 'flush_only_old_table_cache_entries' can't be set to the value of 'FELSE' +set @@global.flush_only_old_table_cache_entries = -1024; +ERROR 42000: Variable 'flush_only_old_table_cache_entries' can't be set to the value of '-1024' +set @@global.flush_only_old_table_cache_entries = 65536; +ERROR 42000: Variable 'flush_only_old_table_cache_entries' can't be set to the value of '65536' +set @@global.flush_only_old_table_cache_entries = 65530.34; +ERROR 42000: Incorrect argument type to variable 'flush_only_old_table_cache_entries' +set @@global.flush_only_old_table_cache_entries = test; +ERROR 42000: Variable 'flush_only_old_table_cache_entries' can't be set to the value of 'test' +set @@global.flush_only_old_table_cache_entries = @start_global_value; diff --git a/mysql-test/suite/sys_vars/t/flush_only_old_table_cache_entries_basic.test b/mysql-test/suite/sys_vars/t/flush_only_old_table_cache_entries_basic.test new file mode 100644 index 000000000000..6ab5d60054fc --- /dev/null +++ b/mysql-test/suite/sys_vars/t/flush_only_old_table_cache_entries_basic.test @@ -0,0 +1,57 @@ +set @start_global_value = @@global.flush_only_old_table_cache_entries; + +select @@global.flush_only_old_table_cache_entries; +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +select @@session.flush_only_old_table_cache_entries; +show global variables like 'flush_only_old_table_cache_entries'; +show session variables like 'flush_only_old_table_cache_entries'; +select * from performance_schema.global_variables where variable_name='flush_only_old_table_cache_entries'; +select * from performance_schema.session_variables where variable_name='flush_only_old_table_cache_entries'; + +# +# show that it's writable +# +set global flush_only_old_table_cache_entries = ON; +select @@global.flush_only_old_table_cache_entries; +--error ER_GLOBAL_VARIABLE +set session flush_only_old_table_cache_entries = ON; + +# +# Change the value of variable to a valid value for GLOBAL Scope # +# +set @@global.flush_only_old_table_cache_entries = ON; +select @@global.flush_only_old_table_cache_entries; +set @@global.flush_only_old_table_cache_entries = OFF; +select @@global.flush_only_old_table_cache_entries; +set @@global.flush_only_old_table_cache_entries = 0; +select @@global.flush_only_old_table_cache_entries; +set @@global.flush_only_old_table_cache_entries = 1; +select @@global.flush_only_old_table_cache_entries; +set @@global.flush_only_old_table_cache_entries = TRUE; +select @@global.flush_only_old_table_cache_entries; +set @@global.flush_only_old_table_cache_entries = FALSE; +select @@global.flush_only_old_table_cache_entries; + +# +# Change the value of flush_only_old_table_cache_entries to an invalid value # +# +--error ER_WRONG_VALUE_FOR_VAR +set @@global.flush_only_old_table_cache_entries = 'ONN'; +--error ER_WRONG_VALUE_FOR_VAR +set @@global.flush_only_old_table_cache_entries = "OFFF"; +--error ER_PARSE_ERROR +set @@global.flush_only_old_table_cache_entries = OF; +--error ER_WRONG_VALUE_FOR_VAR +set @@global.flush_only_old_table_cache_entries = TTRUE; +--error ER_WRONG_VALUE_FOR_VAR +set @@global.flush_only_old_table_cache_entries = FELSE; +--error ER_WRONG_VALUE_FOR_VAR +set @@global.flush_only_old_table_cache_entries = -1024; +--error ER_WRONG_VALUE_FOR_VAR +set @@global.flush_only_old_table_cache_entries = 65536; +--error ER_WRONG_TYPE_FOR_VAR +set @@global.flush_only_old_table_cache_entries = 65530.34; +--error ER_WRONG_VALUE_FOR_VAR +set @@global.flush_only_old_table_cache_entries = test; + +set @@global.flush_only_old_table_cache_entries = @start_global_value; diff --git a/mysql-test/t/all_persisted_variables.test b/mysql-test/t/all_persisted_variables.test index f360dfe89253..95db3228dacd 100644 --- a/mysql-test/t/all_persisted_variables.test +++ b/mysql-test/t/all_persisted_variables.test @@ -39,7 +39,7 @@ --source include/have_binlog_format_row.inc let $total_global_vars=`SELECT COUNT(*) FROM performance_schema.global_variables where variable_name NOT LIKE 'ndb_%'`; -let $total_persistent_vars=398; +let $total_persistent_vars=399; # Due to open bugs, there are fewer variables --let $total_persistent_vars_sans_bugs=`SELECT $total_persistent_vars - 6;` diff --git a/mysql-test/t/flush_only_old_table_cache_entries-master.opt b/mysql-test/t/flush_only_old_table_cache_entries-master.opt new file mode 100644 index 000000000000..48aa67d6c4c4 --- /dev/null +++ b/mysql-test/t/flush_only_old_table_cache_entries-master.opt @@ -0,0 +1 @@ +--flush-time=1 diff --git a/mysql-test/t/flush_only_old_table_cache_entries.test b/mysql-test/t/flush_only_old_table_cache_entries.test new file mode 100644 index 000000000000..e8c6bdffb38c --- /dev/null +++ b/mysql-test/t/flush_only_old_table_cache_entries.test @@ -0,0 +1,16 @@ +--source include/have_debug.inc +--source include/have_debug_sync.inc + +SET @start_global_value = @@global.flush_only_old_table_cache_entries; +SET @@global.debug= '+d,tdc_flush_unused_tables'; + +# Check if flush_only_old_table_cache_entries is disabled +SET debug_sync = 'now WAIT_FOR signal.disabled'; + +# Check if flush_only_old_table_cache_entries is enabled +SET @@global.flush_only_old_table_cache_entries = true; +SET debug_sync = 'now WAIT_FOR signal.enabled'; + +# Clean up +SET @@global.debug= '-d,tdc_flush_unused_tables'; +SET @@global.flush_only_old_table_cache_entries = @start_global_value; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 0a99160e9394..2f1b346b2401 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -1076,6 +1076,7 @@ bool opt_log_unsafe_statements; bool opt_log_global_var_changes; bool is_slave = false; bool read_only_slave; +bool flush_only_old_table_cache_entries = false; #ifdef HAVE_INITGROUPS volatile sig_atomic_t calling_initgroups = 0; /**< Used in SIGSEGV handler. */ diff --git a/sql/mysqld.h b/sql/mysqld.h index 49c8014c4b7a..c3e62688f598 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -389,6 +389,7 @@ inline ulonglong microseconds_to_my_timer(double when) { extern bool is_slave; extern bool read_only_slave; +extern bool flush_only_old_table_cache_entries; extern ulong stored_program_cache_size; extern ulong back_log; extern "C" MYSQL_PLUGIN_IMPORT ulong server_id; diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 75e28bd683db..6ba2ee6b63eb 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -537,6 +537,9 @@ static TABLE_SHARE *process_found_table_share(THD *thd MY_ATTRIBUTE((unused)), share->prev = 0; } + /* update access time */ + share->set_last_access_time(); + /* Free cache if too big */ while (table_def_cache->size() > table_def_size && oldest_unused_share->next) table_def_cache->erase(to_string(oldest_unused_share->table_cache_key)); @@ -833,6 +836,9 @@ TABLE_SHARE *get_table_share(THD *thd, const char *db, const char *table_name, share->m_psi = NULL; #endif + /* update access time */ + share->set_last_access_time(); + DBUG_PRINT("exit", ("share: %p ref_count: %u", share, share->ref_count())); /* If debug, assert that the share is actually present in the cache */ @@ -9531,7 +9537,34 @@ bool mysql_rm_tmp_tables(void) { void tdc_flush_unused_tables() { table_cache_manager.lock_all_and_tdc(); - table_cache_manager.free_all_unused_tables(); + + if (flush_only_old_table_cache_entries) { + DBUG_EXECUTE_IF("tdc_flush_unused_tables", { + static constexpr char act[] = "now SIGNAL signal.enabled"; + DBUG_ASSERT(!debug_sync_set_action(current_thd, STRING_WITH_LEN(act))); + };); + + const time_point flush_cutpoint = + std::chrono::system_clock::now() - std::chrono::seconds(flush_time); + table_cache_manager.free_old_unused_tables(flush_cutpoint); + + /* Free table shares which were not freed implicitly by loop above. */ + for (TABLE_SHARE *s = oldest_unused_share; s->next; s = s->next) { + if (should_be_evicted(s->last_accessed, flush_cutpoint)) { + table_def_cache->erase(to_string(s->table_cache_key)); + } + } + } else { + DBUG_EXECUTE_IF("tdc_flush_unused_tables", { + static constexpr char act[] = "now SIGNAL signal.disabled"; + DBUG_ASSERT(!debug_sync_set_action(current_thd, STRING_WITH_LEN(act))); + };); + + table_cache_manager.free_all_unused_tables(); + /* Free table shares which were not freed implicitly by loop above. */ + while (oldest_unused_share->next) + table_def_cache->erase(to_string(oldest_unused_share->table_cache_key)); + } table_cache_manager.unlock_all_and_tdc(); } diff --git a/sql/sql_manager.cc b/sql/sql_manager.cc index e91a3c9c72c9..a1afd861d123 100644 --- a/sql/sql_manager.cc +++ b/sql/sql_manager.cc @@ -50,6 +50,7 @@ #include "sql/log.h" #include "sql/mysqld.h" // flush_time #include "sql/sql_base.h" // tdc_flush_unused_tables +#include "sql/sql_class.h" static bool volatile manager_thread_in_use; static bool abort_manager; @@ -63,9 +64,15 @@ static void *handle_manager(void *arg MY_ATTRIBUTE((unused))) { int error = 0; struct timespec abstime; bool reset_flush_time = true; + my_thread_init(); DBUG_ENTER("handle_manager"); + THD *thd = new THD; + thd->set_new_thread_id(); + thd->thread_stack = (char *)&thd; + thd->store_globals(); + manager_thread = my_thread_self(); manager_thread_in_use = 1; @@ -95,6 +102,8 @@ static void *handle_manager(void *arg MY_ATTRIBUTE((unused))) { } } manager_thread_in_use = 0; + thd->release_resources(); + delete thd; DBUG_LEAVE; // Can't use DBUG_RETURN after my_thread_end my_thread_end(); return (NULL); diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 738becce192d..da4e06092d64 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -1866,6 +1866,13 @@ static Sys_var_ulong Sys_flush_time( GLOBAL_VAR(flush_time), CMD_LINE(REQUIRED_ARG), VALID_RANGE(0, LONG_TIMEOUT), DEFAULT(0), BLOCK_SIZE(1)); +static Sys_var_bool Sys_flush_only_old_cache_entries( + "flush_only_old_table_cache_entries", + "Enable/disable flushing table and definition cache entries " + "policy based on TTL specified by flush_time.", + GLOBAL_VAR(flush_only_old_table_cache_entries), CMD_LINE(OPT_ARG), + DEFAULT(false)); + static bool check_ftb_syntax(sys_var *, THD *, set_var *var) { return ft_boolean_check_syntax_string( (uchar *)(var->save_result.string_value.str)); diff --git a/sql/table.cc b/sql/table.cc index cd6237fcdf69..64fd75f8b414 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -3137,6 +3137,9 @@ int open_table_from_share(THD *thd, TABLE_SHARE *share, const char *alias, /* Increment the opened_tables counter, only when open flags set. */ if (db_stat) thd->status_var.opened_tables++; + /* set creation time */ + outparam->set_last_access_time(); + DBUG_RETURN(0); err: @@ -7774,3 +7777,15 @@ void TABLE::set_pos_in_table_list(TABLE_LIST *table_list) noexcept { } ////////////////////////////////////////////////////////////////////////// + +void TABLE_SHARE::set_last_access_time() noexcept { + this->last_accessed = std::chrono::system_clock::now(); +} + +void TABLE::set_last_access_time() noexcept { + this->last_accessed = std::chrono::system_clock::now(); +} + +bool should_be_evicted(time_point last_accessed, time_point cutpoint) noexcept { + return last_accessed < cutpoint; +} diff --git a/sql/table.h b/sql/table.h index 24c38d900f27..2ac9f07c9d0e 100644 --- a/sql/table.h +++ b/sql/table.h @@ -25,6 +25,7 @@ #include #include +#include // last access time #include #include "binary_log_types.h" @@ -119,6 +120,8 @@ typedef Mem_root_array_YY Create_col_name_list; typedef int64 query_id_t; +using time_point = std::chrono::system_clock::time_point; + enum class enum_json_diff_operation; bool assert_ref_count_is_locked(const TABLE_SHARE *); @@ -931,6 +934,9 @@ struct TABLE_SHARE { uint foreign_key_parents{0}; TABLE_SHARE_FOREIGN_KEY_PARENT_INFO *foreign_key_parent{nullptr}; + /* last time table_share was accessed via get_table_share() function */ + time_point last_accessed; + /** Set share's table cache key and update its db and table name appropriately. @@ -1091,6 +1097,9 @@ struct TABLE_SHARE { /** Release resources and free memory occupied by the table share. */ void destroy(); + /* reset time for TTL based LRU eviction policy */ + void set_last_access_time() noexcept; + /** How many TABLE objects use this TABLE_SHARE. @return the reference count @@ -1593,6 +1602,9 @@ struct TABLE { bool master_had_triggers{false}; bool disable_sql_log_bin_triggers{false}; + /* last time table was accessed via get_table() function */ + time_point last_accessed; + private: /// Cost model object for operations on this table Cost_model_table m_cost_model; @@ -2119,6 +2131,9 @@ struct TABLE { void prepare_triggers_for_insert_stmt_or_event(); bool prepare_triggers_for_delete_stmt_or_event(); bool prepare_triggers_for_update_stmt_or_event(); + + /* reset time for TTL based LRU eviction policy */ + void set_last_access_time() noexcept; }; static inline void empty_record(TABLE *table) { @@ -3974,4 +3989,6 @@ bool create_table_share_for_upgrade(THD *thd, const char *path, bool is_fix_view_cols_and_deps); ////////////////////////////////////////////////////////////////////////// +bool should_be_evicted(time_point last_accessed, time_point cutpoint) noexcept; + #endif /* TABLE_INCLUDED */ diff --git a/sql/table_cache.cc b/sql/table_cache.cc index ccec630066de..b22854fe0449 100644 --- a/sql/table_cache.cc +++ b/sql/table_cache.cc @@ -129,6 +129,26 @@ void Table_cache::free_all_unused_tables() { } } +void Table_cache::free_old_unused_tables(time_point cutpoint) { + assert_owner(); + + if (!m_unused_tables) return; + std::vector old_tables; + + for (TABLE *t = m_unused_tables;;) { + if (should_be_evicted(t->last_accessed, cutpoint)) { + old_tables.push_back(t); + } + t = t->next; + if (t == m_unused_tables) // circular double-linked list + break; + } + for (TABLE *table_to_free : old_tables) { + remove_table(table_to_free); + intern_close_table(table_to_free); + } +} + #ifndef DBUG_OFF /** Print debug information for the contents of the table cache. @@ -324,6 +344,13 @@ void Table_cache_manager::free_all_unused_tables() { m_table_cache[i].free_all_unused_tables(); } +void Table_cache_manager::free_old_unused_tables(time_point cutpoint) { + assert_owner_all_and_tdc(); + + for (uint i = 0; i < table_cache_instances; i++) + m_table_cache[i].free_old_unused_tables(cutpoint); +} + #ifndef DBUG_OFF /** Print debug information for the contents of all table cache instances. diff --git a/sql/table_cache.h b/sql/table_cache.h index d0575a94c0db..5af9ce44f444 100644 --- a/sql/table_cache.h +++ b/sql/table_cache.h @@ -155,6 +155,7 @@ class Table_cache { uint cached_tables() const { return m_table_count; } void free_all_unused_tables(); + void free_old_unused_tables(time_point cutpoint); #ifndef DBUG_OFF void print_tables(); @@ -197,6 +198,7 @@ class Table_cache_manager { TABLE_SHARE *share); void free_all_unused_tables(); + void free_old_unused_tables(time_point cutpoint); #ifndef DBUG_OFF void print_tables(); @@ -477,6 +479,9 @@ TABLE *Table_cache::get_table(THD *thd, const char *key, size_t key_length, DBUG_ASSERT(table->db_stat && table->file); /* The children must be detached from the table. */ DBUG_ASSERT(!table->file->extra(HA_EXTRA_IS_ATTACHED_CHILDREN)); + + // update access time + table->set_last_access_time(); } return table;