[mysql-5.6][PR] FB8-180: Implemented TTL base cache eviction policy f…

…or "open table" and "table description" caches Summary: Jira issue: https://jira.percona.com/browse/FB8-180 Reference Patch: ec9e122 MySQL has variable "flush_time". If it is set to a nonzero value, all tables are closed every "flush_time" seconds to free up resources and synchronize unflushed data to disk. This option is best used only on systems with minimal resources. I added variable "flush_only_old_cache_entries" which were open more than "flush_time" seconds. Originally Reviewed By: anirbanr-fb GitHub Author: Volodymyr Verovkin <[email protected]> Test Plan: Imported from GitHub, without a `Test Plan:` line. Reviewers: Subscribers: Differential Revision: https://phabricator.intern.facebook.com/D14884057 Tasks: Tags: Blame Revision:
facebook · Apr 30, 2019 · 1bd4eb8 · 1bd4eb8
1 parent 7cac65b
commit 1bd4eb8
Show file tree

Hide file tree

Showing 17 changed files with 273 additions and 6 deletions.
diff --git a/mysql-test/r/all_persisted_variables.result b/mysql-test/r/all_persisted_variables.result
@@ -44,17 +44,17 @@ include/assert.inc ['Expect 500+ variables in the table. Due to open Bugs, we ar
 
 # Test SET PERSIST
 
-include/assert.inc ['Expect 398 persisted variables in the table. Due to open Bugs, we are checking for 392']
+include/assert.inc ['Expect 399 persisted variables in the table. Due to open Bugs, we are checking for 393']
 
 ************************************************************
 * 3. Restart server, it must preserve the persisted variable
 *    settings. Verify persisted configuration.
 ************************************************************
 # restart
 
-include/assert.inc ['Expect 392 persisted variables in persisted_variables table.']
-include/assert.inc ['Expect 392 persisted variables shown as PERSISTED in variables_info table.']
-include/assert.inc ['Expect 392 persisted variables with matching peristed and global values.']
+include/assert.inc ['Expect 393 persisted variables in persisted_variables table.']
+include/assert.inc ['Expect 393 persisted variables shown as PERSISTED in variables_info table.']
+include/assert.inc ['Expect 393 persisted variables with matching peristed and global values.']
 
 ************************************************************
 * 4. Test RESET PERSIST IF EXISTS. Verify persisted variable

diff --git a/mysql-test/r/flush_only_old_table_cache_entries.result b/mysql-test/r/flush_only_old_table_cache_entries.result
@@ -0,0 +1,7 @@
+SET @start_global_value = @@global.flush_only_old_table_cache_entries;
+SET @@global.debug= '+d,tdc_flush_unused_tables';
+SET debug_sync = 'now WAIT_FOR signal.disabled';
+SET @@global.flush_only_old_table_cache_entries = true;
+SET debug_sync = 'now WAIT_FOR signal.enabled';
+SET @@global.debug= '-d,tdc_flush_unused_tables';
+SET @@global.flush_only_old_table_cache_entries = @start_global_value;
diff --git a/mysql-test/r/mysqld--help-notwin.result b/mysql-test/r/mysqld--help-notwin.result
@@ -330,6 +330,9 @@ The following options may be given as the first argument:
  The max size of a file to use for filesort. Raise an
  error when this is exceeded. 0 means no limit.
  --flush             Flush MyISAM tables to disk between SQL commands
+ --flush-only-old-table-cache-entries 
+ Enable/disable flushing table and definition cache
+ entries policy based on TTL specified by flush_time.
  --flush-time=#      A dedicated thread is created to flush all tables at the
  given interval
  --ft-boolean-syntax=name 
@@ -1501,6 +1504,7 @@ explicit-defaults-for-timestamp TRUE
 external-locking FALSE
 filesort-max-file-size 0
 flush FALSE
+flush-only-old-table-cache-entries FALSE
 flush-time 0
 ft-boolean-syntax + -><()~*:""&|
 ft-max-word-len 84

diff --git a/mysql-test/suite/sys_vars/r/flush_only_old_table_cache_entries_basic.result b/mysql-test/suite/sys_vars/r/flush_only_old_table_cache_entries_basic.result
@@ -0,0 +1,67 @@
+set @start_global_value = @@global.flush_only_old_table_cache_entries;
+select @@global.flush_only_old_table_cache_entries;
+@@global.flush_only_old_table_cache_entries
+0
+select @@session.flush_only_old_table_cache_entries;
+ERROR HY000: Variable 'flush_only_old_table_cache_entries' is a GLOBAL variable
+show global variables like 'flush_only_old_table_cache_entries';
+Variable_name	Value
+flush_only_old_table_cache_entries	OFF
+show session variables like 'flush_only_old_table_cache_entries';
+Variable_name	Value
+flush_only_old_table_cache_entries	OFF
+select * from performance_schema.global_variables where variable_name='flush_only_old_table_cache_entries';
+VARIABLE_NAME	VARIABLE_VALUE
+flush_only_old_table_cache_entries	OFF
+select * from performance_schema.session_variables where variable_name='flush_only_old_table_cache_entries';
+VARIABLE_NAME	VARIABLE_VALUE
+flush_only_old_table_cache_entries	OFF
+set global flush_only_old_table_cache_entries = ON;
+select @@global.flush_only_old_table_cache_entries;
+@@global.flush_only_old_table_cache_entries
+1
+set session flush_only_old_table_cache_entries = ON;
+ERROR HY000: Variable 'flush_only_old_table_cache_entries' is a GLOBAL variable and should be set with SET GLOBAL
+set @@global.flush_only_old_table_cache_entries = ON;
+select @@global.flush_only_old_table_cache_entries;
+@@global.flush_only_old_table_cache_entries
+1
+set @@global.flush_only_old_table_cache_entries = OFF;
+select @@global.flush_only_old_table_cache_entries;
+@@global.flush_only_old_table_cache_entries
+0
+set @@global.flush_only_old_table_cache_entries = 0;
+select @@global.flush_only_old_table_cache_entries;
+@@global.flush_only_old_table_cache_entries
+0
+set @@global.flush_only_old_table_cache_entries = 1;
+select @@global.flush_only_old_table_cache_entries;
+@@global.flush_only_old_table_cache_entries
+1
+set @@global.flush_only_old_table_cache_entries = TRUE;
+select @@global.flush_only_old_table_cache_entries;
+@@global.flush_only_old_table_cache_entries
+1
+set @@global.flush_only_old_table_cache_entries = FALSE;
+select @@global.flush_only_old_table_cache_entries;
+@@global.flush_only_old_table_cache_entries
+0
+set @@global.flush_only_old_table_cache_entries = 'ONN';
+ERROR 42000: Variable 'flush_only_old_table_cache_entries' can't be set to the value of 'ONN'
+set @@global.flush_only_old_table_cache_entries = "OFFF";
+ERROR 42000: Variable 'flush_only_old_table_cache_entries' can't be set to the value of 'OFFF'
+set @@global.flush_only_old_table_cache_entries = OF;
+ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'OF' at line 1
+set @@global.flush_only_old_table_cache_entries = TTRUE;
+ERROR 42000: Variable 'flush_only_old_table_cache_entries' can't be set to the value of 'TTRUE'
+set @@global.flush_only_old_table_cache_entries = FELSE;
+ERROR 42000: Variable 'flush_only_old_table_cache_entries' can't be set to the value of 'FELSE'
+set @@global.flush_only_old_table_cache_entries = -1024;
+ERROR 42000: Variable 'flush_only_old_table_cache_entries' can't be set to the value of '-1024'
+set @@global.flush_only_old_table_cache_entries = 65536;
+ERROR 42000: Variable 'flush_only_old_table_cache_entries' can't be set to the value of '65536'
+set @@global.flush_only_old_table_cache_entries = 65530.34;
+ERROR 42000: Incorrect argument type to variable 'flush_only_old_table_cache_entries'
+set @@global.flush_only_old_table_cache_entries = test;
+ERROR 42000: Variable 'flush_only_old_table_cache_entries' can't be set to the value of 'test'
+set @@global.flush_only_old_table_cache_entries = @start_global_value;
diff --git a/mysql-test/suite/sys_vars/t/flush_only_old_table_cache_entries_basic.test b/mysql-test/suite/sys_vars/t/flush_only_old_table_cache_entries_basic.test
@@ -0,0 +1,57 @@
+set @start_global_value = @@global.flush_only_old_table_cache_entries;
+
+select @@global.flush_only_old_table_cache_entries;
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+select @@session.flush_only_old_table_cache_entries;
+show global variables like 'flush_only_old_table_cache_entries';
+show session variables like 'flush_only_old_table_cache_entries';
+select * from performance_schema.global_variables where variable_name='flush_only_old_table_cache_entries';
+select * from performance_schema.session_variables where variable_name='flush_only_old_table_cache_entries';
+
+#
+# show that it's writable
+#
+set global flush_only_old_table_cache_entries = ON;
+select @@global.flush_only_old_table_cache_entries;
+--error ER_GLOBAL_VARIABLE
+set session flush_only_old_table_cache_entries = ON;
+
+#
+# Change the value of variable to a valid value for GLOBAL Scope #
+#
+set @@global.flush_only_old_table_cache_entries = ON;
+select @@global.flush_only_old_table_cache_entries;
+set @@global.flush_only_old_table_cache_entries = OFF;
+select @@global.flush_only_old_table_cache_entries;
+set @@global.flush_only_old_table_cache_entries = 0;
+select @@global.flush_only_old_table_cache_entries;
+set @@global.flush_only_old_table_cache_entries = 1;
+select @@global.flush_only_old_table_cache_entries;
+set @@global.flush_only_old_table_cache_entries = TRUE;
+select @@global.flush_only_old_table_cache_entries;
+set @@global.flush_only_old_table_cache_entries = FALSE;
+select @@global.flush_only_old_table_cache_entries;
+
+#
+# Change the value of flush_only_old_table_cache_entries to an invalid value #
+#
+--error ER_WRONG_VALUE_FOR_VAR
+set @@global.flush_only_old_table_cache_entries = 'ONN';
+--error ER_WRONG_VALUE_FOR_VAR
+set @@global.flush_only_old_table_cache_entries = "OFFF";
+--error ER_PARSE_ERROR
+set @@global.flush_only_old_table_cache_entries = OF;
+--error ER_WRONG_VALUE_FOR_VAR
+set @@global.flush_only_old_table_cache_entries = TTRUE;
+--error ER_WRONG_VALUE_FOR_VAR
+set @@global.flush_only_old_table_cache_entries = FELSE;
+--error ER_WRONG_VALUE_FOR_VAR
+set @@global.flush_only_old_table_cache_entries = -1024;
+--error ER_WRONG_VALUE_FOR_VAR
+set @@global.flush_only_old_table_cache_entries = 65536;
+--error ER_WRONG_TYPE_FOR_VAR
+set @@global.flush_only_old_table_cache_entries = 65530.34;
+--error ER_WRONG_VALUE_FOR_VAR
+set @@global.flush_only_old_table_cache_entries = test;
+
+set @@global.flush_only_old_table_cache_entries = @start_global_value;
diff --git a/mysql-test/t/all_persisted_variables.test b/mysql-test/t/all_persisted_variables.test
@@ -39,7 +39,7 @@
 --source include/have_binlog_format_row.inc
 
 let $total_global_vars=`SELECT COUNT(*) FROM performance_schema.global_variables where variable_name NOT LIKE 'ndb_%'`;
-let $total_persistent_vars=398;
+let $total_persistent_vars=399;
 # Due to open bugs, there are fewer variables
 --let $total_persistent_vars_sans_bugs=`SELECT $total_persistent_vars - 6;`
 

diff --git a/mysql-test/t/flush_only_old_table_cache_entries-master.opt b/mysql-test/t/flush_only_old_table_cache_entries-master.opt
@@ -0,0 +1 @@
+--flush-time=1
diff --git a/mysql-test/t/flush_only_old_table_cache_entries.test b/mysql-test/t/flush_only_old_table_cache_entries.test
@@ -0,0 +1,16 @@
+--source include/have_debug.inc
+--source include/have_debug_sync.inc
+
+SET @start_global_value = @@global.flush_only_old_table_cache_entries;
+SET @@global.debug= '+d,tdc_flush_unused_tables';
+
+# Check if flush_only_old_table_cache_entries is disabled
+SET debug_sync = 'now WAIT_FOR signal.disabled';
+
+# Check if flush_only_old_table_cache_entries is enabled
+SET @@global.flush_only_old_table_cache_entries = true;
+SET debug_sync = 'now WAIT_FOR signal.enabled';
+
+# Clean up
+SET @@global.debug= '-d,tdc_flush_unused_tables';
+SET @@global.flush_only_old_table_cache_entries = @start_global_value;
diff --git a/sql/mysqld.cc b/sql/mysqld.cc
@@ -1076,6 +1076,7 @@ bool opt_log_unsafe_statements;
 bool opt_log_global_var_changes;
 bool is_slave = false;
 bool read_only_slave;
+bool flush_only_old_table_cache_entries = false;
 
 #ifdef HAVE_INITGROUPS
 volatile sig_atomic_t calling_initgroups = 0; /**< Used in SIGSEGV handler. */

diff --git a/sql/mysqld.h b/sql/mysqld.h
@@ -389,6 +389,7 @@ inline ulonglong microseconds_to_my_timer(double when) {
 
 extern bool is_slave;
 extern bool read_only_slave;
+extern bool flush_only_old_table_cache_entries;
 extern ulong stored_program_cache_size;
 extern ulong back_log;
 extern "C" MYSQL_PLUGIN_IMPORT ulong server_id;

diff --git a/sql/sql_base.cc b/sql/sql_base.cc
@@ -537,6 +537,9 @@ static TABLE_SHARE *process_found_table_share(THD *thd MY_ATTRIBUTE((unused)),
     share->prev = 0;
   }
 
+  /* update access time */
+  share->set_last_access_time();
+
   /* Free cache if too big */
   while (table_def_cache->size() > table_def_size && oldest_unused_share->next)
     table_def_cache->erase(to_string(oldest_unused_share->table_cache_key));
@@ -833,6 +836,9 @@ TABLE_SHARE *get_table_share(THD *thd, const char *db, const char *table_name,
   share->m_psi = NULL;
 #endif
 
+  /* update access time */
+  share->set_last_access_time();
+
   DBUG_PRINT("exit", ("share: %p  ref_count: %u", share, share->ref_count()));
 
   /* If debug, assert that the share is actually present in the cache */
@@ -9531,7 +9537,34 @@ bool mysql_rm_tmp_tables(void) {
 
 void tdc_flush_unused_tables() {
   table_cache_manager.lock_all_and_tdc();
-  table_cache_manager.free_all_unused_tables();
+
+  if (flush_only_old_table_cache_entries) {
+    DBUG_EXECUTE_IF("tdc_flush_unused_tables", {
+      static constexpr char act[] = "now SIGNAL signal.enabled";
+      DBUG_ASSERT(!debug_sync_set_action(current_thd, STRING_WITH_LEN(act)));
+    };);
+
+    const time_point flush_cutpoint =
+        std::chrono::system_clock::now() - std::chrono::seconds(flush_time);
+    table_cache_manager.free_old_unused_tables(flush_cutpoint);
+
+    /* Free table shares which were not freed implicitly by loop above. */
+    for (TABLE_SHARE *s = oldest_unused_share; s->next; s = s->next) {
+      if (should_be_evicted(s->last_accessed, flush_cutpoint)) {
+        table_def_cache->erase(to_string(s->table_cache_key));
+      }
+    }
+  } else {
+    DBUG_EXECUTE_IF("tdc_flush_unused_tables", {
+      static constexpr char act[] = "now SIGNAL signal.disabled";
+      DBUG_ASSERT(!debug_sync_set_action(current_thd, STRING_WITH_LEN(act)));
+    };);
+
+    table_cache_manager.free_all_unused_tables();
+    /* Free table shares which were not freed implicitly by loop above. */
+    while (oldest_unused_share->next)
+      table_def_cache->erase(to_string(oldest_unused_share->table_cache_key));
+  }
   table_cache_manager.unlock_all_and_tdc();
 }
 

diff --git a/sql/sql_manager.cc b/sql/sql_manager.cc
@@ -50,6 +50,7 @@
 #include "sql/log.h"
 #include "sql/mysqld.h"    // flush_time
 #include "sql/sql_base.h"  // tdc_flush_unused_tables
+#include "sql/sql_class.h"
 
 static bool volatile manager_thread_in_use;
 static bool abort_manager;
@@ -63,9 +64,15 @@ static void *handle_manager(void *arg MY_ATTRIBUTE((unused))) {
   int error = 0;
   struct timespec abstime;
   bool reset_flush_time = true;
+
   my_thread_init();
   DBUG_ENTER("handle_manager");
 
+  THD *thd = new THD;
+  thd->set_new_thread_id();
+  thd->thread_stack = (char *)&thd;
+  thd->store_globals();
+
   manager_thread = my_thread_self();
   manager_thread_in_use = 1;
 
@@ -95,6 +102,8 @@ static void *handle_manager(void *arg MY_ATTRIBUTE((unused))) {
     }
   }
   manager_thread_in_use = 0;
+  thd->release_resources();
+  delete thd;
   DBUG_LEAVE;  // Can't use DBUG_RETURN after my_thread_end
   my_thread_end();
   return (NULL);

diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc
@@ -1866,6 +1866,13 @@ static Sys_var_ulong Sys_flush_time(
     GLOBAL_VAR(flush_time), CMD_LINE(REQUIRED_ARG),
     VALID_RANGE(0, LONG_TIMEOUT), DEFAULT(0), BLOCK_SIZE(1));
 
+static Sys_var_bool Sys_flush_only_old_cache_entries(
+    "flush_only_old_table_cache_entries",
+    "Enable/disable flushing table and definition cache entries "
+    "policy based on TTL specified by flush_time.",
+    GLOBAL_VAR(flush_only_old_table_cache_entries), CMD_LINE(OPT_ARG),
+    DEFAULT(false));
+
 static bool check_ftb_syntax(sys_var *, THD *, set_var *var) {
   return ft_boolean_check_syntax_string(
       (uchar *)(var->save_result.string_value.str));

diff --git a/sql/table.cc b/sql/table.cc
@@ -3137,6 +3137,9 @@ int open_table_from_share(THD *thd, TABLE_SHARE *share, const char *alias,
   /* Increment the opened_tables counter, only when open flags set. */
   if (db_stat) thd->status_var.opened_tables++;
 
+  /* set creation time */
+  outparam->set_last_access_time();
+
   DBUG_RETURN(0);
 
 err:
@@ -7774,3 +7777,15 @@ void TABLE::set_pos_in_table_list(TABLE_LIST *table_list) noexcept {
 }
 
 //////////////////////////////////////////////////////////////////////////
+
+void TABLE_SHARE::set_last_access_time() noexcept {
+  this->last_accessed = std::chrono::system_clock::now();
+}
+
+void TABLE::set_last_access_time() noexcept {
+  this->last_accessed = std::chrono::system_clock::now();
+}
+
+bool should_be_evicted(time_point last_accessed, time_point cutpoint) noexcept {
+  return last_accessed < cutpoint;
+}