[mysql-5.6][PR] add capability to manually flush memtable and l0

Summary: This provides an option to flush memtable and remove all files from L0, which can be useful for lessening variance in benchmarking on read-only and read-heavy tests, in cases where the memtable isn't full and/or there aren't many files in L0, and no writes are occurring. See more details at facebook#427. @update-submodule: rocksdb Closes facebook#520 Github Author: Alex Yang <[email protected]> Github PR Sync: {sync, type="child", parent="github", parentrepo="facebook/mysql-5.6", parentprnum="520", parentprfbid="156908684811954"} Test Plan: Imported from GitHub, without a `Test Plan:` line. Reviewers: herman, jkedgar, svcscm Reviewed By: svcscm Subscribers: svcscm, [email protected] Differential Revision: https://phabricator.intern.facebook.com/D4509351 Signature: t1:4509351:1490814923:d177f0faf2cc11c4e76afd5399343a34c152fd74
IslamAbdelRahman · Mar 29, 2017 · 3f04f3c · 3f04f3c
1 parent 3e3c7f2
commit 3f04f3c
Show file tree

Hide file tree

Showing 5 changed files with 121 additions and 1 deletion.
diff --git a/mysql-test/r/mysqld--help-notwin-profiling.result b/mysql-test/r/mysqld--help-notwin-profiling.result
@@ -1160,6 +1160,9 @@ The following options may be given as the first argument:
  --rocksdb-force-compute-memtable-stats 
  Force to always compute memtable stats
  (Defaults to on; use --skip-rocksdb-force-compute-memtable-stats to disable.)
+ --rocksdb-force-flush-memtable-and-lzero-now 
+ Acts similar to force_flush_memtable_now, but also
+ compacts all L0 files.
  --rocksdb-force-flush-memtable-now 
  Forces memstore flush which may block all write requests
  so be careful
@@ -1994,6 +1997,7 @@ rocksdb-error-if-exists FALSE
 rocksdb-flush-log-at-trx-commit 1
 rocksdb-flush-memtable-on-analyze TRUE
 rocksdb-force-compute-memtable-stats TRUE
+rocksdb-force-flush-memtable-and-lzero-now FALSE
 rocksdb-force-flush-memtable-now FALSE
 rocksdb-force-index-records-in-range 0
 rocksdb-global-info ON

diff --git a/mysql-test/r/mysqld--help-notwin.result b/mysql-test/r/mysqld--help-notwin.result
@@ -1158,6 +1158,9 @@ The following options may be given as the first argument:
  --rocksdb-force-compute-memtable-stats 
  Force to always compute memtable stats
  (Defaults to on; use --skip-rocksdb-force-compute-memtable-stats to disable.)
+ --rocksdb-force-flush-memtable-and-lzero-now 
+ Acts similar to force_flush_memtable_now, but also
+ compacts all L0 files.
  --rocksdb-force-flush-memtable-now 
  Forces memstore flush which may block all write requests
  so be careful
@@ -1991,6 +1994,7 @@ rocksdb-error-if-exists FALSE
 rocksdb-flush-log-at-trx-commit 1
 rocksdb-flush-memtable-on-analyze TRUE
 rocksdb-force-compute-memtable-stats TRUE
+rocksdb-force-flush-memtable-and-lzero-now FALSE
 rocksdb-force-flush-memtable-now FALSE
 rocksdb-force-index-records-in-range 0
 rocksdb-global-info ON

diff --git a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_and_lzero_now_basic.result b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_and_lzero_now_basic.result
@@ -0,0 +1,50 @@
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(1);
+INSERT INTO valid_values VALUES(0);
+INSERT INTO valid_values VALUES('on');
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+SET @start_global_value = @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW;
+SELECT @start_global_value;
+@start_global_value
+0
+'# Setting to valid values in global scope#'
+"Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to 1"
+SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW   = 1;
+SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW;
+@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW
+0
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = DEFAULT;
+SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW;
+@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW
+0
+"Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to 0"
+SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW   = 0;
+SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW;
+@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW
+0
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = DEFAULT;
+SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW;
+@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW
+0
+"Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to on"
+SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW   = on;
+SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW;
+@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW
+0
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = DEFAULT;
+SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW;
+@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW
+0
+"Trying to set variable @@session.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to 444. It should fail because it is not session."
+SET @@session.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW   = 444;
+ERROR HY000: Variable 'rocksdb_force_flush_memtable_and_lzero_now' is a GLOBAL variable and should be set with SET GLOBAL
+'# Testing with invalid values in global scope #'
+SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = @start_global_value;
+SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW;
+@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW
+0
+DROP TABLE valid_values;
+DROP TABLE invalid_values;
diff --git a/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_and_lzero_now_basic.test b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_and_lzero_now_basic.test
@@ -0,0 +1,17 @@
+--source include/have_rocksdb.inc
+
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(1);
+INSERT INTO valid_values VALUES(0);
+INSERT INTO valid_values VALUES('on');
+
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+
+--let $sys_var=ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW
+--let $read_only=0
+--let $session=0
+--let $sticky=1
+--source suite/sys_vars/inc/rocksdb_sys_var.inc
+
+DROP TABLE valid_values;
+DROP TABLE invalid_values;
diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc
@@ -283,11 +283,47 @@ static void rocksdb_force_flush_memtable_now_stub(
 static int rocksdb_force_flush_memtable_now(
     THD *const thd, struct st_mysql_sys_var *const var, void *const var_ptr,
     struct st_mysql_value *const value) {
-  sql_print_information("RocksDB: Manual memtable flush\n");
+  sql_print_information("RocksDB: Manual memtable flush.");
   rocksdb_flush_all_memtables();
   return HA_EXIT_SUCCESS;
 }
 
+static void rocksdb_force_flush_memtable_and_lzero_now_stub(
+    THD *const thd, struct st_mysql_sys_var *const var, void *const var_ptr,
+    const void *const save) {}
+
+static int rocksdb_force_flush_memtable_and_lzero_now(
+    THD *const thd, struct st_mysql_sys_var *const var, void *const var_ptr,
+    struct st_mysql_value *const value) {
+  sql_print_information("RocksDB: Manual memtable and L0 flush.");
+  rocksdb_flush_all_memtables();
+
+  const Rdb_cf_manager &cf_manager = rdb_get_cf_manager();
+  const rocksdb::CompactionOptions c_options = rocksdb::CompactionOptions();
+  for (const auto &cf_handle : cf_manager.get_all_cf()) {
+    rocksdb::ColumnFamilyMetaData metadata;
+    rdb->GetColumnFamilyMetaData(cf_handle, &metadata);
+
+    DBUG_ASSERT(metadata.levels[0].level == 0);
+    std::vector<std::string> file_names;
+    for (auto &file : metadata.levels[0].files) {
+      file_names.emplace_back(file.db_path + file.name);
+    }
+
+    if (!file_names.empty()) {
+      rocksdb::Status s;
+      s = rdb->CompactFiles(c_options, cf_handle, file_names, 1);
+
+      if (!s.ok()) {
+        rdb_handle_io_error(s, RDB_IO_ERROR_GENERAL);
+        return HA_EXIT_FAILURE;
+      }
+    }
+  }
+
+  return HA_EXIT_SUCCESS;
+}
+
 static void rocksdb_drop_index_wakeup_thread(
     my_core::THD *const thd MY_ATTRIBUTE((__unused__)),
     struct st_mysql_sys_var *const var MY_ATTRIBUTE((__unused__)),
@@ -385,6 +421,7 @@ static my_bool rocksdb_enable_2pc = 0;
 static char *rocksdb_strict_collation_exceptions;
 static my_bool rocksdb_collect_sst_properties = 1;
 static my_bool rocksdb_force_flush_memtable_now_var = 0;
+static my_bool rocksdb_force_flush_memtable_and_lzero_now_var = 0;
 static uint64_t rocksdb_number_stat_computes = 0;
 static uint32_t rocksdb_seconds_between_stat_computes = 3600;
 static long long rocksdb_compaction_sequential_deletes = 0l;
@@ -1086,6 +1123,13 @@ static MYSQL_SYSVAR_BOOL(
     rocksdb_force_flush_memtable_now, rocksdb_force_flush_memtable_now_stub,
     FALSE);
 
+static MYSQL_SYSVAR_BOOL(
+    force_flush_memtable_and_lzero_now,
+    rocksdb_force_flush_memtable_and_lzero_now_var, PLUGIN_VAR_RQCMDARG,
+    "Acts similar to force_flush_memtable_now, but also compacts all L0 files.",
+    rocksdb_force_flush_memtable_and_lzero_now,
+    rocksdb_force_flush_memtable_and_lzero_now_stub, FALSE);
+
 static MYSQL_THDVAR_BOOL(
     flush_memtable_on_analyze, PLUGIN_VAR_RQCMDARG,
     "Forces memtable flush on ANALZYE table to get accurate cardinality",
@@ -1299,6 +1343,7 @@ static struct st_mysql_sys_var *rocksdb_system_variables[] = {
     MYSQL_SYSVAR(strict_collation_exceptions),
     MYSQL_SYSVAR(collect_sst_properties),
     MYSQL_SYSVAR(force_flush_memtable_now),
+    MYSQL_SYSVAR(force_flush_memtable_and_lzero_now),
     MYSQL_SYSVAR(flush_memtable_on_analyze),
     MYSQL_SYSVAR(seconds_between_stat_computes),