From 67d5a72518fa63203d8d8f7adf92c39029a72bb8 Mon Sep 17 00:00:00 2001 From: Yoshinori Matsunobu Date: Thu, 4 Jan 2018 16:18:38 -0800 Subject: [PATCH] Reducing default rocksdb_max_row_locks from 1B to 1M (#780) (#780) Summary: Default rocksdb_max_row_locks was way too high and caused OOMs, like issue#692. This diff reduces the maximum to 1M. Closes https://github.com/facebook/mysql-5.6/pull/780 Differential Revision: D6661811 Pulled By: asandryh --- .../r/add_index_inplace_sstfilewriter.result | 4 ++++ mysql-test/suite/rocksdb/r/rocksdb.result | 2 +- .../t/add_index_inplace_sstfilewriter.test | 5 +++++ .../r/rocksdb_max_row_locks_basic.result | 18 +++++++++--------- storage/rocksdb/ha_rocksdb.cc | 2 +- 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/mysql-test/suite/rocksdb/r/add_index_inplace_sstfilewriter.result b/mysql-test/suite/rocksdb/r/add_index_inplace_sstfilewriter.result index 2d1ba7ca1d89..ec9c940f250e 100644 --- a/mysql-test/suite/rocksdb/r/add_index_inplace_sstfilewriter.result +++ b/mysql-test/suite/rocksdb/r/add_index_inplace_sstfilewriter.result @@ -15,6 +15,10 @@ count(b) 3000000 ALTER TABLE t1 ADD INDEX kb(b), ALGORITHM=INPLACE; ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; +ERROR HY000: Status error 10 received from RocksDB: Operation aborted: Failed to acquire lock due to max_num_locks limit +set session rocksdb_bulk_load=1; +ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; +set session rocksdb_bulk_load=0; SELECT COUNT(*) as c FROM (SELECT COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `b`, CONCAT(ISNULL(`b`)))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `t1` FORCE INDEX(`kb`) UNION DISTINCT diff --git a/mysql-test/suite/rocksdb/r/rocksdb.result b/mysql-test/suite/rocksdb/r/rocksdb.result index 86238c128349..7fee90a1c851 100644 --- a/mysql-test/suite/rocksdb/r/rocksdb.result +++ b/mysql-test/suite/rocksdb/r/rocksdb.result @@ -925,7 +925,7 @@ rocksdb_max_background_jobs 2 rocksdb_max_latest_deadlocks 5 rocksdb_max_log_file_size 0 rocksdb_max_manifest_file_size 18446744073709551615 -rocksdb_max_row_locks 1073741824 +rocksdb_max_row_locks 1048576 rocksdb_max_subcompactions 1 rocksdb_max_total_wal_size 0 rocksdb_merge_buf_size 67108864 diff --git a/mysql-test/suite/rocksdb/t/add_index_inplace_sstfilewriter.test b/mysql-test/suite/rocksdb/t/add_index_inplace_sstfilewriter.test index 7e600224dcc4..b743b5ca39da 100644 --- a/mysql-test/suite/rocksdb/t/add_index_inplace_sstfilewriter.test +++ b/mysql-test/suite/rocksdb/t/add_index_inplace_sstfilewriter.test @@ -62,7 +62,12 @@ ALTER TABLE t1 ADD INDEX kb(b), ALGORITHM=INPLACE; # disable duplicate index warning --disable_warnings # now do same index using copy algorithm +# hitting max row locks (1M) +--error ER_RDB_STATUS_GENERAL ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; +set session rocksdb_bulk_load=1; +ALTER TABLE t1 ADD INDEX kb_copy(b), ALGORITHM=COPY; +set session rocksdb_bulk_load=0; --enable_warnings # checksum testing diff --git a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_max_row_locks_basic.result b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_max_row_locks_basic.result index e417e4d5c4e9..c925a68d4ed0 100644 --- a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_max_row_locks_basic.result +++ b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_max_row_locks_basic.result @@ -6,11 +6,11 @@ INSERT INTO invalid_values VALUES('\'aaa\''); SET @start_global_value = @@global.ROCKSDB_MAX_ROW_LOCKS; SELECT @start_global_value; @start_global_value -1073741824 +1048576 SET @start_session_value = @@session.ROCKSDB_MAX_ROW_LOCKS; SELECT @start_session_value; @start_session_value -1073741824 +1048576 '# Setting to valid values in global scope#' "Trying to set variable @@global.ROCKSDB_MAX_ROW_LOCKS to 1" SET @@global.ROCKSDB_MAX_ROW_LOCKS = 1; @@ -21,7 +21,7 @@ SELECT @@global.ROCKSDB_MAX_ROW_LOCKS; SET @@global.ROCKSDB_MAX_ROW_LOCKS = DEFAULT; SELECT @@global.ROCKSDB_MAX_ROW_LOCKS; @@global.ROCKSDB_MAX_ROW_LOCKS -1073741824 +1048576 "Trying to set variable @@global.ROCKSDB_MAX_ROW_LOCKS to 1024" SET @@global.ROCKSDB_MAX_ROW_LOCKS = 1024; SELECT @@global.ROCKSDB_MAX_ROW_LOCKS; @@ -31,7 +31,7 @@ SELECT @@global.ROCKSDB_MAX_ROW_LOCKS; SET @@global.ROCKSDB_MAX_ROW_LOCKS = DEFAULT; SELECT @@global.ROCKSDB_MAX_ROW_LOCKS; @@global.ROCKSDB_MAX_ROW_LOCKS -1073741824 +1048576 '# Setting to valid values in session scope#' "Trying to set variable @@session.ROCKSDB_MAX_ROW_LOCKS to 1" SET @@session.ROCKSDB_MAX_ROW_LOCKS = 1; @@ -42,7 +42,7 @@ SELECT @@session.ROCKSDB_MAX_ROW_LOCKS; SET @@session.ROCKSDB_MAX_ROW_LOCKS = DEFAULT; SELECT @@session.ROCKSDB_MAX_ROW_LOCKS; @@session.ROCKSDB_MAX_ROW_LOCKS -1073741824 +1048576 "Trying to set variable @@session.ROCKSDB_MAX_ROW_LOCKS to 1024" SET @@session.ROCKSDB_MAX_ROW_LOCKS = 1024; SELECT @@session.ROCKSDB_MAX_ROW_LOCKS; @@ -52,21 +52,21 @@ SELECT @@session.ROCKSDB_MAX_ROW_LOCKS; SET @@session.ROCKSDB_MAX_ROW_LOCKS = DEFAULT; SELECT @@session.ROCKSDB_MAX_ROW_LOCKS; @@session.ROCKSDB_MAX_ROW_LOCKS -1073741824 +1048576 '# Testing with invalid values in global scope #' "Trying to set variable @@global.ROCKSDB_MAX_ROW_LOCKS to 'aaa'" SET @@global.ROCKSDB_MAX_ROW_LOCKS = 'aaa'; Got one of the listed errors SELECT @@global.ROCKSDB_MAX_ROW_LOCKS; @@global.ROCKSDB_MAX_ROW_LOCKS -1073741824 +1048576 SET @@global.ROCKSDB_MAX_ROW_LOCKS = @start_global_value; SELECT @@global.ROCKSDB_MAX_ROW_LOCKS; @@global.ROCKSDB_MAX_ROW_LOCKS -1073741824 +1048576 SET @@session.ROCKSDB_MAX_ROW_LOCKS = @start_session_value; SELECT @@session.ROCKSDB_MAX_ROW_LOCKS; @@session.ROCKSDB_MAX_ROW_LOCKS -1073741824 +1048576 DROP TABLE valid_values; DROP TABLE invalid_values; diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 622e0adee122..247a1dc46a56 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -614,7 +614,7 @@ static TYPELIB index_type_typelib = {array_elements(index_type_names) - 1, nullptr}; const ulong RDB_MAX_LOCK_WAIT_SECONDS = 1024 * 1024 * 1024; -const ulong RDB_MAX_ROW_LOCKS = 1024 * 1024 * 1024; +const ulong RDB_MAX_ROW_LOCKS = 1024 * 1024; const ulong RDB_DEFAULT_BULK_LOAD_SIZE = 1000; const ulong RDB_MAX_BULK_LOAD_SIZE = 1024 * 1024 * 1024; const size_t RDB_DEFAULT_MERGE_BUF_SIZE = 64 * 1024 * 1024;