Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Simplify RocksDB config options #2855

Merged
merged 1 commit into from
Jul 24, 2020
Merged

Simplify RocksDB config options #2855

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 3 additions & 24 deletions nano/core_test/toml.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,15 +255,8 @@ TEST (toml, daemon_config_deserialize_defaults)
ASSERT_EQ (conf.node.lmdb_config.map_size, defaults.node.lmdb_config.map_size);

ASSERT_EQ (conf.node.rocksdb_config.enable, defaults.node.rocksdb_config.enable);
ASSERT_EQ (conf.node.rocksdb_config.bloom_filter_bits, defaults.node.rocksdb_config.bloom_filter_bits);
ASSERT_EQ (conf.node.rocksdb_config.block_cache, defaults.node.rocksdb_config.block_cache);
ASSERT_EQ (conf.node.rocksdb_config.memory_multiplier, defaults.node.rocksdb_config.memory_multiplier);
ASSERT_EQ (conf.node.rocksdb_config.io_threads, defaults.node.rocksdb_config.io_threads);
ASSERT_EQ (conf.node.rocksdb_config.enable_pipelined_write, defaults.node.rocksdb_config.enable_pipelined_write);
ASSERT_EQ (conf.node.rocksdb_config.cache_index_and_filter_blocks, defaults.node.rocksdb_config.cache_index_and_filter_blocks);
ASSERT_EQ (conf.node.rocksdb_config.block_size, defaults.node.rocksdb_config.block_size);
ASSERT_EQ (conf.node.rocksdb_config.memtable_size, defaults.node.rocksdb_config.memtable_size);
ASSERT_EQ (conf.node.rocksdb_config.num_memtables, defaults.node.rocksdb_config.num_memtables);
ASSERT_EQ (conf.node.rocksdb_config.total_memtable_size, defaults.node.rocksdb_config.total_memtable_size);
}

TEST (toml, optional_child)
Expand Down Expand Up @@ -513,15 +506,8 @@ TEST (toml, daemon_config_deserialize_no_defaults)

[node.rocksdb]
enable = true
bloom_filter_bits = 10
block_cache = 512
memory_multiplier = 3
io_threads = 99
enable_pipelined_write = true
cache_index_and_filter_blocks = true
block_size = 16
memtable_size = 128
num_memtables = 3
total_memtable_size = 0

[node.experimental]
secondary_work_peers = ["test.org:998"]
Expand Down Expand Up @@ -668,15 +654,8 @@ TEST (toml, daemon_config_deserialize_no_defaults)
ASSERT_NE (conf.node.lmdb_config.map_size, defaults.node.lmdb_config.map_size);

ASSERT_NE (conf.node.rocksdb_config.enable, defaults.node.rocksdb_config.enable);
ASSERT_NE (conf.node.rocksdb_config.bloom_filter_bits, defaults.node.rocksdb_config.bloom_filter_bits);
ASSERT_NE (conf.node.rocksdb_config.block_cache, defaults.node.rocksdb_config.block_cache);
ASSERT_NE (conf.node.rocksdb_config.memory_multiplier, defaults.node.rocksdb_config.memory_multiplier);
ASSERT_NE (conf.node.rocksdb_config.io_threads, defaults.node.rocksdb_config.io_threads);
ASSERT_NE (conf.node.rocksdb_config.enable_pipelined_write, defaults.node.rocksdb_config.enable_pipelined_write);
ASSERT_NE (conf.node.rocksdb_config.cache_index_and_filter_blocks, defaults.node.rocksdb_config.cache_index_and_filter_blocks);
ASSERT_NE (conf.node.rocksdb_config.block_size, defaults.node.rocksdb_config.block_size);
ASSERT_NE (conf.node.rocksdb_config.memtable_size, defaults.node.rocksdb_config.memtable_size);
ASSERT_NE (conf.node.rocksdb_config.num_memtables, defaults.node.rocksdb_config.num_memtables);
ASSERT_NE (conf.node.rocksdb_config.total_memtable_size, defaults.node.rocksdb_config.total_memtable_size);
}

/** There should be no required values **/
Expand Down
38 changes: 4 additions & 34 deletions nano/lib/rocksdbconfig.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,55 +4,25 @@
nano::error nano::rocksdb_config::serialize_toml (nano::tomlconfig & toml) const
{
toml.put ("enable", enable, "Whether to use the RocksDB backend for the ledger database.\ntype:bool");
toml.put ("enable_pipelined_write", enable_pipelined_write, "Whether to use 2 separate write queues for memtable/WAL, true is recommended.\ntype:bool");
toml.put ("cache_index_and_filter_blocks", cache_index_and_filter_blocks, "Whether index and filter blocks are stored in block_cache, true is recommended.\ntype:bool");
toml.put ("bloom_filter_bits", bloom_filter_bits, "Number of bits to use with a bloom filter. Helps with point reads but uses more memory. 0 disables the bloom filter, 10 is recommended.\ntype:uint32");
toml.put ("block_cache", block_cache, "Size (MB) of the block cache; A larger number will increase performance of read operations. At least 512MB is recommended.\ntype:uint64");
toml.put ("memory_multiplier", memory_multiplier, "This will modify how much memory is used represented by 1 (low), 2 (medium), 3 (high). Default is 2.\ntype:uint8");
toml.put ("io_threads", io_threads, "Number of threads to use with the background compaction and flushing. Number of hardware threads is recommended.\ntype:uint32");
toml.put ("block_size", block_size, "Uncompressed data (KBs) per block. Increasing block size decreases memory usage and space amplification, but increases read amplification. 16 is recommended.\ntype:uint32");
toml.put ("num_memtables", num_memtables, "Number of memtables to keep in memory per column family. 2 is the minimum, 3 is recommended.\ntype:uint32");
toml.put ("memtable_size", memtable_size, "Amount of memory (MB) to build up before flushing to disk for an individual column family. Large values increase performance. 64 or 128 is recommended.\ntype:uint32");
toml.put ("total_memtable_size", total_memtable_size, "Total memory (MB) which can be used across all memtables, set to 0 for unconstrained.\ntype:uint32");
return toml.get_error ();
}

nano::error nano::rocksdb_config::deserialize_toml (nano::tomlconfig & toml)
{
toml.get_optional<bool> ("enable", enable);
toml.get_optional<bool> ("enable_pipelined_write", enable_pipelined_write);
toml.get_optional<bool> ("cache_index_and_filter_blocks", cache_index_and_filter_blocks);
toml.get_optional<unsigned> ("bloom_filter_bits", bloom_filter_bits);
toml.get_optional<uint64_t> ("block_cache", block_cache);
toml.get_optional<uint8_t> ("memory_multiplier", memory_multiplier);
toml.get_optional<unsigned> ("io_threads", io_threads);
toml.get_optional<unsigned> ("block_size", block_size);
toml.get_optional<unsigned> ("num_memtables", num_memtables);
toml.get_optional<unsigned> ("memtable_size", memtable_size);
toml.get_optional<unsigned> ("total_memtable_size", total_memtable_size);

// Validate ranges
if (bloom_filter_bits > 100)
{
toml.get_error ().set ("bloom_filter_bits is too high");
}
if (num_memtables < 2)
{
toml.get_error ().set ("num_memtables must be at least 2");
}
if (memtable_size == 0)
{
toml.get_error ().set ("memtable_size must be non-zero");
}
if ((total_memtable_size < memtable_size * 8) && (total_memtable_size != 0))
{
toml.get_error ().set ("total_memtable_size should be at least 8 times greater than memtable_size or be set to 0");
}
if (io_threads == 0)
{
toml.get_error ().set ("io_threads must be non-zero");
}
if (block_size == 0)
if (memory_multiplier < 1 || memory_multiplier > 3)
{
toml.get_error ().set ("block_size must be non-zero");
toml.get_error ().set ("memory_multiplier must be either 1, 2 or 3");
}

return toml.get_error ();
Expand Down
9 changes: 1 addition & 8 deletions nano/lib/rocksdbconfig.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,7 @@ class rocksdb_config final
nano::error deserialize_toml (nano::tomlconfig & toml_a);

bool enable{ false };
unsigned bloom_filter_bits{ 0 };
uint64_t block_cache{ 64 }; // MB
uint8_t memory_multiplier{ 2 };
unsigned io_threads{ std::thread::hardware_concurrency () };
bool enable_pipelined_write{ false };
bool cache_index_and_filter_blocks{ false };
unsigned block_size{ 4 }; // KB
unsigned memtable_size{ 32 }; // MB
unsigned num_memtables{ 2 }; // Need a minimum of 2
unsigned total_memtable_size{ 512 }; // MB
};
}
31 changes: 12 additions & 19 deletions nano/node/rocksdb/rocksdb.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -473,10 +473,7 @@ rocksdb::Options nano::rocksdb_store::get_db_options () const
db_options.OptimizeLevelStyleCompaction ();

// Adds a separate write queue for memtable/WAL
db_options.enable_pipelined_write = rocksdb_config.enable_pipelined_write;

// Total size of memtables across column families. This can be used to manage the total memory used by memtables.
db_options.db_write_buffer_size = rocksdb_config.total_memtable_size * 1024 * 1024ULL;
db_options.enable_pipelined_write = true;

return db_options;
}
Expand All @@ -486,21 +483,17 @@ rocksdb::BlockBasedTableOptions nano::rocksdb_store::get_table_options () const
rocksdb::BlockBasedTableOptions table_options;

// Block cache for reads
table_options.block_cache = rocksdb::NewLRUCache (rocksdb_config.block_cache * 1024 * 1024ULL);
table_options.block_cache = rocksdb::NewLRUCache (1024ULL * 1024 * base_block_cache_size * rocksdb_config.memory_multiplier);

// Bloom filter to help with point reads
auto bloom_filter_bits = rocksdb_config.bloom_filter_bits;
if (bloom_filter_bits > 0)
{
table_options.filter_policy.reset (rocksdb::NewBloomFilterPolicy (bloom_filter_bits, false));
}
auto bloom_filter_bits = 10;
table_options.filter_policy.reset (rocksdb::NewBloomFilterPolicy (bloom_filter_bits, false));

// Increasing block_size decreases memory usage and space amplification, but increases read amplification.
table_options.block_size = rocksdb_config.block_size * 1024ULL;
table_options.block_size = 16 * 1024ULL;

// Whether index and filter blocks are stored in block_cache. These settings should be synced
table_options.cache_index_and_filter_blocks = rocksdb_config.cache_index_and_filter_blocks;
table_options.pin_l0_filter_and_index_blocks_in_cache = rocksdb_config.cache_index_and_filter_blocks;
// Whether index and filter blocks are stored in block_cache
table_options.pin_l0_filter_and_index_blocks_in_cache = true;

return table_options;
}
Expand All @@ -514,7 +507,7 @@ rocksdb::ColumnFamilyOptions nano::rocksdb_store::get_cf_options () const
cf_options.level0_file_num_compaction_trigger = 4;

// L1 size, compaction is triggered for L0 at this size (4 SST files in L1)
cf_options.max_bytes_for_level_base = 1024ULL * 1024 * 4 * rocksdb_config.memtable_size;
cf_options.max_bytes_for_level_base = 1024ULL * 1024 * 4 * rocksdb_config.memory_multiplier * base_memtable_size;

// Each level is a multiple of the above. If L1 is 512MB. L2 will be 512 * 8 = 2GB. L3 will be 2GB * 8 = 16GB, and so on...
cf_options.max_bytes_for_level_multiplier = 8;
Expand All @@ -523,16 +516,16 @@ rocksdb::ColumnFamilyOptions nano::rocksdb_store::get_cf_options () const
cf_options.ttl = 1 * 24 * 60 * 60;

// Size of level 1 sst files
cf_options.target_file_size_base = 1024ULL * 1024 * rocksdb_config.memtable_size;
cf_options.target_file_size_base = 1024ULL * 1024 * rocksdb_config.memory_multiplier * base_memtable_size;

// Size of each memtable
cf_options.write_buffer_size = 1024ULL * 1024 * rocksdb_config.memtable_size;
cf_options.write_buffer_size = 1024ULL * 1024 * rocksdb_config.memory_multiplier * base_memtable_size;

// Size target of levels are changed dynamically based on size of the last level
cf_options.level_compaction_dynamic_level_bytes = true;

// Number of memtables to keep in memory (1 active, rest inactive/immutable)
cf_options.max_write_buffer_number = rocksdb_config.num_memtables;
// Number of memtables to keep in memory (1 active, 1 inactive)
cf_options.max_write_buffer_number = 2;

return cf_options;
}
Expand Down
3 changes: 3 additions & 0 deletions nano/node/rocksdb/rocksdb.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,9 @@ class rocksdb_store : public block_store_partial<rocksdb::Slice, rocksdb_store>
rocksdb::Options get_db_options () const;
rocksdb::BlockBasedTableOptions get_table_options () const;
nano::rocksdb_config rocksdb_config;

constexpr static int base_memtable_size = 16;
constexpr static int base_block_cache_size = 16;
};

extern template class block_store_partial<rocksdb::Slice, rocksdb_store>;
Expand Down