Introducing a dedicated thread for handling manual compactions in MyR…

…ocks

Upstream commit ID : fb-mysql-5.6.35/566b7f095eb1d62a6379c33c911f36d7434898e7
PS-5217 : Merge fb-prod201803

Summary:
SET GLOBAL rocksdb_compact_cf='cf_name' is a command to trigger
a manual compaction. Manual compaction may take a long time, depending
on target column family size. Prior to this diff, when running manual
compaction with SET GLOBAL, clean mysqld shutdown was blocked until the
manual compaction finished. This was because shutdown path had no way to
terminate a user thread calling CompactRange.

This diff changes how to handle manual compactions. Changes are as follows.

- Introducing a dedicated MyRocks thread to handle manual compactions (via SET
  GLOBAL rocksdb_compact_cf). Inside RocksDB, only one manual compaction can be
  running at a time so concurrency does not change.
- Adding a system variable (rocksdb_max_manual_compactions), a debug system
  variable, and a session variable (rocksdb_manual_compaction_threads), and a
  few status variables to track manual compaction progress.

With this diff, normal shutdown no longer waits for manual compaction to finish.
Also, you can change the number of manual compaction threads based on each
compaction. For example, you can increase compaction threads to 16 for large
CFs, and uses 1 thread for small CFs.

A user thread calling SET GLOBAL rocksdb_compact_cf checks the manual
compaction thread activities, as well as its own thd state. If its manual
compaction finishes, or its own thd is killed, the user thread proceeds. As a
result, mysqld normal shutdown is no longer blocked by long running manual
compactions.

Manual compaction thread does not persist any state.

This diff does not change OPTIMIZE TABLE behavior. It will be covered by future
diffs.

Reviewed By: lth

Differential Revision: D8874346

fbshipit-source-id: 396f787ff78

Percona Server:
- Fixed issue with debug delay variable where code depended on specific expected
  behavior of my_sleep w.r.t. shutdown is different in PS 5.7.
- Moved testing into its own eplicit test file and fixed test to use proper mtr
  idioms.
- Slightly touched up commit comment formatting.

mysql-test/suite/rocksdb.sys_vars/r/rocksdb_debug_manual_compaction_delay_basic.result

@@ -0,0 +1,46 @@
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(2400);
+INSERT INTO valid_values VALUES(100000);
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO invalid_values VALUES('\'aaa\'');
+SET @start_global_value = @@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY;
+SELECT @start_global_value;
+@start_global_value
+0
+'# Setting to valid values in global scope#'
+"Trying to set variable @@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY to 2400"
+SET @@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY   = 2400;
+SELECT @@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY;
+@@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY
+2400
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY = DEFAULT;
+SELECT @@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY;
+@@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY
+0
+"Trying to set variable @@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY to 100000"
+SET @@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY   = 100000;
+SELECT @@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY;
+@@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY
+100000
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY = DEFAULT;
+SELECT @@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY;
+@@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY
+0
+"Trying to set variable @@session.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY to 444. It should fail because it is not session."
+SET @@session.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY   = 444;
+ERROR HY000: Variable 'rocksdb_debug_manual_compaction_delay' is a GLOBAL variable and should be set with SET GLOBAL
+'# Testing with invalid values in global scope #'
+"Trying to set variable @@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY to 'aaa'"
+SET @@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY   = 'aaa';
+Got one of the listed errors
+SELECT @@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY;
+@@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY
+0
+SET @@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY = @start_global_value;
+SELECT @@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY;
+@@global.ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY
+0
+DROP TABLE valid_values;
+DROP TABLE invalid_values;

mysql-test/suite/rocksdb.sys_vars/r/rocksdb_manual_compaction_threads_basic.result

@@ -0,0 +1,93 @@
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(0);
+INSERT INTO valid_values VALUES(1);
+INSERT INTO valid_values VALUES(99);
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO invalid_values VALUES('\'aaa\'');
+SET @start_global_value = @@global.ROCKSDB_MANUAL_COMPACTION_THREADS;
+SELECT @start_global_value;
+@start_global_value
+0
+SET @start_session_value = @@session.ROCKSDB_MANUAL_COMPACTION_THREADS;
+SELECT @start_session_value;
+@start_session_value
+0
+'# Setting to valid values in global scope#'
+"Trying to set variable @@global.ROCKSDB_MANUAL_COMPACTION_THREADS to 0"
+SET @@global.ROCKSDB_MANUAL_COMPACTION_THREADS   = 0;
+SELECT @@global.ROCKSDB_MANUAL_COMPACTION_THREADS;
+@@global.ROCKSDB_MANUAL_COMPACTION_THREADS
+0
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_MANUAL_COMPACTION_THREADS = DEFAULT;
+SELECT @@global.ROCKSDB_MANUAL_COMPACTION_THREADS;
+@@global.ROCKSDB_MANUAL_COMPACTION_THREADS
+0
+"Trying to set variable @@global.ROCKSDB_MANUAL_COMPACTION_THREADS to 1"
+SET @@global.ROCKSDB_MANUAL_COMPACTION_THREADS   = 1;
+SELECT @@global.ROCKSDB_MANUAL_COMPACTION_THREADS;
+@@global.ROCKSDB_MANUAL_COMPACTION_THREADS
+1
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_MANUAL_COMPACTION_THREADS = DEFAULT;
+SELECT @@global.ROCKSDB_MANUAL_COMPACTION_THREADS;
+@@global.ROCKSDB_MANUAL_COMPACTION_THREADS
+0
+"Trying to set variable @@global.ROCKSDB_MANUAL_COMPACTION_THREADS to 99"
+SET @@global.ROCKSDB_MANUAL_COMPACTION_THREADS   = 99;
+SELECT @@global.ROCKSDB_MANUAL_COMPACTION_THREADS;
+@@global.ROCKSDB_MANUAL_COMPACTION_THREADS
+99
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_MANUAL_COMPACTION_THREADS = DEFAULT;
+SELECT @@global.ROCKSDB_MANUAL_COMPACTION_THREADS;
+@@global.ROCKSDB_MANUAL_COMPACTION_THREADS
+0
+'# Setting to valid values in session scope#'
+"Trying to set variable @@session.ROCKSDB_MANUAL_COMPACTION_THREADS to 0"
+SET @@session.ROCKSDB_MANUAL_COMPACTION_THREADS   = 0;
+SELECT @@session.ROCKSDB_MANUAL_COMPACTION_THREADS;
+@@session.ROCKSDB_MANUAL_COMPACTION_THREADS
+0
+"Setting the session scope variable back to default"
+SET @@session.ROCKSDB_MANUAL_COMPACTION_THREADS = DEFAULT;
+SELECT @@session.ROCKSDB_MANUAL_COMPACTION_THREADS;
+@@session.ROCKSDB_MANUAL_COMPACTION_THREADS
+0
+"Trying to set variable @@session.ROCKSDB_MANUAL_COMPACTION_THREADS to 1"
+SET @@session.ROCKSDB_MANUAL_COMPACTION_THREADS   = 1;
+SELECT @@session.ROCKSDB_MANUAL_COMPACTION_THREADS;
+@@session.ROCKSDB_MANUAL_COMPACTION_THREADS
+1
+"Setting the session scope variable back to default"
+SET @@session.ROCKSDB_MANUAL_COMPACTION_THREADS = DEFAULT;
+SELECT @@session.ROCKSDB_MANUAL_COMPACTION_THREADS;
+@@session.ROCKSDB_MANUAL_COMPACTION_THREADS
+0
+"Trying to set variable @@session.ROCKSDB_MANUAL_COMPACTION_THREADS to 99"
+SET @@session.ROCKSDB_MANUAL_COMPACTION_THREADS   = 99;
+SELECT @@session.ROCKSDB_MANUAL_COMPACTION_THREADS;
+@@session.ROCKSDB_MANUAL_COMPACTION_THREADS
+99
+"Setting the session scope variable back to default"
+SET @@session.ROCKSDB_MANUAL_COMPACTION_THREADS = DEFAULT;
+SELECT @@session.ROCKSDB_MANUAL_COMPACTION_THREADS;
+@@session.ROCKSDB_MANUAL_COMPACTION_THREADS
+0
+'# Testing with invalid values in global scope #'
+"Trying to set variable @@global.ROCKSDB_MANUAL_COMPACTION_THREADS to 'aaa'"
+SET @@global.ROCKSDB_MANUAL_COMPACTION_THREADS   = 'aaa';
+Got one of the listed errors
+SELECT @@global.ROCKSDB_MANUAL_COMPACTION_THREADS;
+@@global.ROCKSDB_MANUAL_COMPACTION_THREADS
+0
+SET @@global.ROCKSDB_MANUAL_COMPACTION_THREADS = @start_global_value;
+SELECT @@global.ROCKSDB_MANUAL_COMPACTION_THREADS;
+@@global.ROCKSDB_MANUAL_COMPACTION_THREADS
+0
+SET @@session.ROCKSDB_MANUAL_COMPACTION_THREADS = @start_session_value;
+SELECT @@session.ROCKSDB_MANUAL_COMPACTION_THREADS;
+@@session.ROCKSDB_MANUAL_COMPACTION_THREADS
+0
+DROP TABLE valid_values;
+DROP TABLE invalid_values;

mysql-test/suite/rocksdb.sys_vars/r/rocksdb_max_manual_compactions_basic.result

@@ -0,0 +1,57 @@
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(1);
+INSERT INTO valid_values VALUES(1024);
+INSERT INTO valid_values VALUES(512*1024*1024);
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO invalid_values VALUES('\'aaa\'');
+SET @start_global_value = @@global.ROCKSDB_MAX_MANUAL_COMPACTIONS;
+SELECT @start_global_value;
+@start_global_value
+10
+'# Setting to valid values in global scope#'
+"Trying to set variable @@global.ROCKSDB_MAX_MANUAL_COMPACTIONS to 1"
+SET @@global.ROCKSDB_MAX_MANUAL_COMPACTIONS   = 1;
+SELECT @@global.ROCKSDB_MAX_MANUAL_COMPACTIONS;
+@@global.ROCKSDB_MAX_MANUAL_COMPACTIONS
+1
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_MAX_MANUAL_COMPACTIONS = DEFAULT;
+SELECT @@global.ROCKSDB_MAX_MANUAL_COMPACTIONS;
+@@global.ROCKSDB_MAX_MANUAL_COMPACTIONS
+10
+"Trying to set variable @@global.ROCKSDB_MAX_MANUAL_COMPACTIONS to 1024"
+SET @@global.ROCKSDB_MAX_MANUAL_COMPACTIONS   = 1024;
+SELECT @@global.ROCKSDB_MAX_MANUAL_COMPACTIONS;
+@@global.ROCKSDB_MAX_MANUAL_COMPACTIONS
+1024
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_MAX_MANUAL_COMPACTIONS = DEFAULT;
+SELECT @@global.ROCKSDB_MAX_MANUAL_COMPACTIONS;
+@@global.ROCKSDB_MAX_MANUAL_COMPACTIONS
+10
+"Trying to set variable @@global.ROCKSDB_MAX_MANUAL_COMPACTIONS to 536870912"
+SET @@global.ROCKSDB_MAX_MANUAL_COMPACTIONS   = 536870912;
+SELECT @@global.ROCKSDB_MAX_MANUAL_COMPACTIONS;
+@@global.ROCKSDB_MAX_MANUAL_COMPACTIONS
+536870912
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_MAX_MANUAL_COMPACTIONS = DEFAULT;
+SELECT @@global.ROCKSDB_MAX_MANUAL_COMPACTIONS;
+@@global.ROCKSDB_MAX_MANUAL_COMPACTIONS
+10
+"Trying to set variable @@session.ROCKSDB_MAX_MANUAL_COMPACTIONS to 444. It should fail because it is not session."
+SET @@session.ROCKSDB_MAX_MANUAL_COMPACTIONS   = 444;
+ERROR HY000: Variable 'rocksdb_max_manual_compactions' is a GLOBAL variable and should be set with SET GLOBAL
+'# Testing with invalid values in global scope #'
+"Trying to set variable @@global.ROCKSDB_MAX_MANUAL_COMPACTIONS to 'aaa'"
+SET @@global.ROCKSDB_MAX_MANUAL_COMPACTIONS   = 'aaa';
+Got one of the listed errors
+SELECT @@global.ROCKSDB_MAX_MANUAL_COMPACTIONS;
+@@global.ROCKSDB_MAX_MANUAL_COMPACTIONS
+10
+SET @@global.ROCKSDB_MAX_MANUAL_COMPACTIONS = @start_global_value;
+SELECT @@global.ROCKSDB_MAX_MANUAL_COMPACTIONS;
+@@global.ROCKSDB_MAX_MANUAL_COMPACTIONS
+10
+DROP TABLE valid_values;
+DROP TABLE invalid_values;

mysql-test/suite/rocksdb.sys_vars/t/rocksdb_debug_manual_compaction_delay_basic.test

@@ -0,0 +1,16 @@
+--source include/have_rocksdb.inc
+
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(2400);
+INSERT INTO valid_values VALUES(100000);
+
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO invalid_values VALUES('\'aaa\'');
+
+--let $sys_var=ROCKSDB_DEBUG_MANUAL_COMPACTION_DELAY
+--let $read_only=0
+--let $session=0
+--source ../include/rocksdb_sys_var.inc
+
+DROP TABLE valid_values;
+DROP TABLE invalid_values;

mysql-test/suite/rocksdb.sys_vars/t/rocksdb_manual_compaction_threads_basic.test

@@ -0,0 +1,17 @@
+--source include/have_rocksdb.inc
+
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(0);
+INSERT INTO valid_values VALUES(1);
+INSERT INTO valid_values VALUES(99);
+
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO invalid_values VALUES('\'aaa\'');
+
+--let $sys_var=ROCKSDB_MANUAL_COMPACTION_THREADS
+--let $read_only=0
+--let $session=1
+--source ../include/rocksdb_sys_var.inc
+
+DROP TABLE valid_values;
+DROP TABLE invalid_values;

mysql-test/suite/rocksdb.sys_vars/t/rocksdb_max_manual_compactions_basic.test

@@ -0,0 +1,17 @@
+--source include/have_rocksdb.inc
+
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(1);
+INSERT INTO valid_values VALUES(1024);
+INSERT INTO valid_values VALUES(512*1024*1024);
+
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO invalid_values VALUES('\'aaa\'');
+
+--let $sys_var=ROCKSDB_MAX_MANUAL_COMPACTIONS
+--let $read_only=0
+--let $session=0
+--source ../include/rocksdb_sys_var.inc
+
+DROP TABLE valid_values;
+DROP TABLE invalid_values;

mysql-test/suite/rocksdb/r/non_blocking_manual_compaction.result

@@ -0,0 +1,18 @@
+CREATE TABLE t1 (
+a int not null,
+b int not null,
+primary key (a,b) comment 'cf1',
+key (b) comment 'rev:cf2'
+) ENGINE=ROCKSDB;
+DELETE FROM t1;
+SET GLOBAL rocksdb_max_manual_compactions = 2;
+SET GLOBAL rocksdb_debug_manual_compaction_delay = 3600;
+SET GLOBAL rocksdb_compact_cf='cf1';
+SET GLOBAL rocksdb_compact_cf='rev:cf2';
+Timeout in wait_condition.inc for select variable_value > 1 from performance_schema.global_status where variable_name='rocksdb_manual_compactions_running'
+SET GLOBAL rocksdb_compact_cf='cf1';
+ERROR HY000: Internal error: Can't schedule more manual compactions. Increase rocksdb_max_manual_compactions or stop issuing more manual compactions.
+SET GLOBAL rocksdb_compact_cf='rev:cf2';
+ERROR HY000: Internal error: Can't schedule more manual compactions. Increase rocksdb_max_manual_compactions or stop issuing more manual compactions.
+DROP TABLE t1;
+# restart

mysql-test/suite/rocksdb/r/rocksdb.result

@@ -922,6 +922,7 @@ rocksdb_datadir	./.rocksdb
 rocksdb_db_write_buffer_size	0
 rocksdb_deadlock_detect	OFF
 rocksdb_deadlock_detect_depth	50
+rocksdb_debug_manual_compaction_delay	0
 rocksdb_debug_optimizer_no_zero_cardinality	ON
 rocksdb_debug_ttl_ignore_pk	OFF
 rocksdb_debug_ttl_read_filter_ts	0
@@ -954,11 +955,13 @@ rocksdb_lock_scanned_rows	OFF
 rocksdb_lock_wait_timeout	1
 rocksdb_log_file_time_to_roll	0
 rocksdb_manifest_preallocation_size	4194304
+rocksdb_manual_compaction_threads	0
 rocksdb_manual_wal_flush	ON
 rocksdb_max_background_jobs	2
 rocksdb_max_latest_deadlocks	5
 rocksdb_max_log_file_size	0
 rocksdb_max_manifest_file_size	1073741824
+rocksdb_max_manual_compactions	10
 rocksdb_max_row_locks	1048576
 rocksdb_max_subcompactions	1
 rocksdb_max_total_wal_size	2147483648
@@ -1568,6 +1571,8 @@ rocksdb_get_hit_l1	#
 rocksdb_get_hit_l2_and_up	#
 rocksdb_get_updates_since_calls	#
 rocksdb_iter_bytes_read	#
+rocksdb_manual_compactions_processed	#
+rocksdb_manual_compactions_running	#
 rocksdb_memtable_hit	#
 rocksdb_memtable_miss	#
 rocksdb_no_file_closes	#
@@ -1676,6 +1681,8 @@ rocksdb_get_hit_l1
 rocksdb_get_hit_l2_and_up
 rocksdb_get_updates_since_calls
 rocksdb_iter_bytes_read
+rocksdb_manual_compactions_processed
+rocksdb_manual_compactions_running
 rocksdb_memtable_hit
 rocksdb_memtable_miss
 rocksdb_no_file_closes
@@ -1786,6 +1793,8 @@ rocksdb_get_hit_l1
 rocksdb_get_hit_l2_and_up
 rocksdb_get_updates_since_calls
 rocksdb_iter_bytes_read
+rocksdb_manual_compactions_processed
+rocksdb_manual_compactions_running
 rocksdb_memtable_hit
 rocksdb_memtable_miss
 rocksdb_no_file_closes

mysql-test/suite/rocksdb/t/non_blocking_manual_compaction-master.opt

@@ -0,0 +1,2 @@
+--loose-rocksdb_max_subcompactions=1
+--loose-rocksdb_default_cf_options=write_buffer_size=512k;target_file_size_base=512k;level0_file_num_compaction_trigger=2;level0_slowdown_writes_trigger=-1;level0_stop_writes_trigger=1000;max_bytes_for_level_base=1m

mysql-test/suite/rocksdb/t/non_blocking_manual_compaction.test

@@ -0,0 +1,48 @@
+--source include/have_rocksdb.inc
+--source include/count_sessions.inc
+
+CREATE TABLE t1 (
+  a int not null,
+  b int not null,
+  primary key (a,b) comment 'cf1',
+  key (b) comment 'rev:cf2'
+) ENGINE=ROCKSDB;
+
+# Populate tables
+let $max = 1000;
+let $table = t1;
+--source suite/rocksdb/include/drop_table_repopulate_table.inc
+
+# Run manual compaction, then restarting mysqld
+# and confirming it is not blocked.
+SET GLOBAL rocksdb_max_manual_compactions = 2;
+SET GLOBAL rocksdb_debug_manual_compaction_delay = 3600;
+connect (con1, localhost, root,,);
+connect (con2, localhost, root,,);
+connect (con3, localhost, root,,);
+connection con1;
+send SET GLOBAL rocksdb_compact_cf='cf1';
+connection con2;
+send SET GLOBAL rocksdb_compact_cf='rev:cf2';
+connection default;
+let $wait_condition = select count(*) = 2 from information_schema.processlist where info like 'SET GLOBAL rocksdb_compact_cf%';
+--source include/wait_condition.inc
+
+let $wait_condition = select variable_value > 1 from performance_schema.global_status where variable_name='rocksdb_manual_compactions_running';
+--source include/wait_condition.inc
+
+connection con3;
+--error ER_INTERNAL_ERROR
+SET GLOBAL rocksdb_compact_cf='cf1';
+--error ER_INTERNAL_ERROR
+SET GLOBAL rocksdb_compact_cf='rev:cf2';
+
+connection default;
+disconnect con1;
+disconnect con2;
+disconnect con3;
+
+DROP TABLE t1;
+
+--source include/restart_mysqld.inc
+--source include/wait_until_count_sessions.inc