-
Notifications
You must be signed in to change notification settings - Fork 3.8k
* improve for sharding #6958
* improve for sharding #6958
Changes from all commits
3fddb72
a7fc838
3dd25b4
1476688
5b79745
dd79cb7
1a14e9c
f7ecd76
74a143c
0e78c1f
bbf1a35
de26672
23b06b7
f2d4539
a8c01c8
2aa4bed
77b91a1
0cbccbb
e093cfa
56726b0
250e33f
fd77886
f97f0de
3d05285
88a2d6a
c8f2607
d995779
22457bf
3b15c4f
60dcf1a
c699544
99f7704
f5e4529
a234e21
7c1e14a
a8c1ced
00a29f3
12d9961
b41c77d
0776c11
1d48532
cb95019
517fd2c
9958d83
bf85b4f
a827e9d
5e8e294
5a9da91
15902ac
90120e2
03b2320
e245e07
c83c212
a29d257
222ef47
6845170
ebebb90
686f0de
c33ae26
4b94016
21cfa55
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
+31 −27 | application.cpp | |
+4 −0 | include/appbase/application.hpp | |
+11 −9 | include/appbase/execution_priority_queue.hpp |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -1427,39 +1427,44 @@ void mongo_db_plugin_impl::init() { | |
} | ||
|
||
try { | ||
// Due to the vast amounts of data, we suggest MongoDB administrators: | ||
// 1. enableSharding database (default to EOS) | ||
// 2. shardCollection: blocks, action_traces, transaction_traces, especially action_traces | ||
// 3. Use compound index with shard key (default to _id), to improve query performance. | ||
|
||
// blocks indexes | ||
auto blocks = mongo_conn[db_name][blocks_col]; | ||
blocks.create_index( bsoncxx::from_json( R"xxx({ "block_num" : 1 })xxx" )); | ||
blocks.create_index( bsoncxx::from_json( R"xxx({ "block_id" : 1 })xxx" )); | ||
blocks.create_index( bsoncxx::from_json( R"xxx({ "block_num" : 1, "_id" : 1 })xxx" )); | ||
blocks.create_index( bsoncxx::from_json( R"xxx({ "block_id" : 1, "_id" : 1 })xxx" )); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do you have any feel for how much larger the indexes are when _id is added? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
shows:
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I wonder if the plugin should even create any indexes and let that be left to the user. However, I'm inclined to accept this PR if you rebase to Thanks. |
||
|
||
auto block_states = mongo_conn[db_name][block_states_col]; | ||
block_states.create_index( bsoncxx::from_json( R"xxx({ "block_num" : 1 })xxx" )); | ||
block_states.create_index( bsoncxx::from_json( R"xxx({ "block_id" : 1 })xxx" )); | ||
block_states.create_index( bsoncxx::from_json( R"xxx({ "block_num" : 1, "_id" : 1 })xxx" )); | ||
block_states.create_index( bsoncxx::from_json( R"xxx({ "block_id" : 1, "_id" : 1 })xxx" )); | ||
|
||
// accounts indexes | ||
accounts.create_index( bsoncxx::from_json( R"xxx({ "name" : 1 })xxx" )); | ||
accounts.create_index( bsoncxx::from_json( R"xxx({ "name" : 1, "_id" : 1 })xxx" )); | ||
|
||
// transactions indexes | ||
auto trans = mongo_conn[db_name][trans_col]; | ||
trans.create_index( bsoncxx::from_json( R"xxx({ "trx_id" : 1 })xxx" )); | ||
trans.create_index( bsoncxx::from_json( R"xxx({ "trx_id" : 1, "_id" : 1 })xxx" )); | ||
|
||
auto trans_trace = mongo_conn[db_name][trans_traces_col]; | ||
trans_trace.create_index( bsoncxx::from_json( R"xxx({ "id" : 1 })xxx" )); | ||
trans_trace.create_index( bsoncxx::from_json( R"xxx({ "id" : 1, "_id" : 1 })xxx" )); | ||
|
||
// action traces indexes | ||
auto action_traces = mongo_conn[db_name][action_traces_col]; | ||
action_traces.create_index( bsoncxx::from_json( R"xxx({ "block_num" : 1 })xxx" )); | ||
action_traces.create_index( bsoncxx::from_json( R"xxx({ "block_num" : 1, "_id" : 1 })xxx" )); | ||
|
||
// pub_keys indexes | ||
auto pub_keys = mongo_conn[db_name][pub_keys_col]; | ||
pub_keys.create_index( bsoncxx::from_json( R"xxx({ "account" : 1, "permission" : 1 })xxx" )); | ||
pub_keys.create_index( bsoncxx::from_json( R"xxx({ "public_key" : 1 })xxx" )); | ||
pub_keys.create_index( bsoncxx::from_json( R"xxx({ "account" : 1, "permission" : 1, "_id" : 1 })xxx" )); | ||
pub_keys.create_index( bsoncxx::from_json( R"xxx({ "public_key" : 1, "_id" : 1 })xxx" )); | ||
|
||
// account_controls indexes | ||
auto account_controls = mongo_conn[db_name][account_controls_col]; | ||
account_controls.create_index( | ||
bsoncxx::from_json( R"xxx({ "controlled_account" : 1, "controlled_permission" : 1 })xxx" )); | ||
account_controls.create_index( bsoncxx::from_json( R"xxx({ "controlling_account" : 1 })xxx" )); | ||
bsoncxx::from_json( R"xxx({ "controlled_account" : 1, "controlled_permission" : 1, "_id" : 1 })xxx" )); | ||
account_controls.create_index( bsoncxx::from_json( R"xxx({ "controlling_account" : 1, "_id" : 1 })xxx" )); | ||
|
||
} catch (...) { | ||
handle_mongo_exception( "create indexes", __LINE__ ); | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,131 @@ | ||
class Boost < Formula | ||
desc "Collection of portable C++ source libraries" | ||
homepage "https://www.boost.org/" | ||
revision 1 | ||
head "https://github.com/boostorg/boost.git" | ||
|
||
stable do | ||
url "https://dl.bintray.com/boostorg/release/1.67.0/source/boost_1_67_0.tar.bz2" | ||
sha256 "2684c972994ee57fc5632e03bf044746f6eb45d4920c343937a465fd67a5adba" | ||
|
||
# Remove for > 1.67.0 | ||
# Fix "error: no member named 'next' in namespace 'boost'" | ||
# Upstream commit from 1 Dec 2017 "Add #include <boost/next_prior.hpp>; no | ||
# longer in utility.hpp" | ||
patch :p2 do | ||
url "https://github.com/boostorg/lockfree/commit/12726cd.patch?full_index=1" | ||
sha256 "f165823d961a588b622b20520668b08819eb5fdc08be7894c06edce78026ce0a" | ||
end | ||
end | ||
|
||
bottle do | ||
cellar :any | ||
sha256 "265ab8beaa6fa26a7c305ef2e6aec8bd26ca1db105aca0aaca028f32c5245a90" => :high_sierra | ||
sha256 "567f3e9a294413c1701b698d666a521cfdeec846e256c6e66576d5b70eb26f08" => :sierra | ||
sha256 "3f3f687a620f656fe2ac54f01306e00e6bbc0e9797db284a8d272648d427e640" => :el_capitan | ||
end | ||
|
||
option "with-icu4c", "Build regexp engine with icu support" | ||
option "without-single", "Disable building single-threading variant" | ||
option "without-static", "Disable building static library variant" | ||
|
||
deprecated_option "with-icu" => "with-icu4c" | ||
|
||
depends_on "icu4c" => :optional | ||
|
||
def install | ||
# Force boost to compile with the desired compiler | ||
open("user-config.jam", "a") do |file| | ||
file.write "using darwin : : #{ENV.cxx} ;\n" | ||
end | ||
|
||
# libdir should be set by --prefix but isn't | ||
bootstrap_args = ["--prefix=#{prefix}", "--libdir=#{lib}"] | ||
|
||
if build.with? "icu4c" | ||
icu4c_prefix = Formula["icu4c"].opt_prefix | ||
bootstrap_args << "--with-icu=#{icu4c_prefix}" | ||
else | ||
bootstrap_args << "--without-icu" | ||
end | ||
|
||
# Handle libraries that will not be built. | ||
without_libraries = ["python", "mpi"] | ||
|
||
# Boost.Log cannot be built using Apple GCC at the moment. Disabled | ||
# on such systems. | ||
without_libraries << "log" if ENV.compiler == :gcc | ||
|
||
bootstrap_args << "--without-libraries=#{without_libraries.join(",")}" | ||
|
||
# layout should be synchronized with boost-python and boost-mpi | ||
args = ["--prefix=#{prefix}", | ||
"--libdir=#{lib}", | ||
"-d2", | ||
"-j#{ENV.make_jobs}", | ||
"--layout=tagged", | ||
"--user-config=user-config.jam", | ||
"-sNO_LZMA=1", | ||
"install"] | ||
|
||
if build.with? "single" | ||
args << "threading=multi,single" | ||
else | ||
args << "threading=multi" | ||
end | ||
|
||
if build.with? "static" | ||
args << "link=shared,static" | ||
else | ||
args << "link=shared" | ||
end | ||
|
||
# Trunk starts using "clang++ -x c" to select C compiler which breaks C++11 | ||
# handling using ENV.cxx11. Using "cxxflags" and "linkflags" still works. | ||
args << "cxxflags=-std=c++11" | ||
if ENV.compiler == :clang | ||
args << "cxxflags=-stdlib=libc++" << "linkflags=-stdlib=libc++" | ||
end | ||
|
||
system "./bootstrap.sh", *bootstrap_args | ||
system "./b2", "headers" | ||
system "./b2", *args | ||
end | ||
|
||
def caveats | ||
s = "" | ||
# ENV.compiler doesn't exist in caveats. Check library availability | ||
# instead. | ||
if Dir["#{lib}/libboost_log*"].empty? | ||
s += <<~EOS | ||
Building of Boost.Log is disabled because it requires newer GCC or Clang. | ||
EOS | ||
end | ||
|
||
s | ||
end | ||
|
||
test do | ||
(testpath/"test.cpp").write <<~EOS | ||
#include <boost/algorithm/string.hpp> | ||
#include <string> | ||
#include <vector> | ||
#include <assert.h> | ||
using namespace boost::algorithm; | ||
using namespace std; | ||
|
||
int main() | ||
{ | ||
string str("a,b"); | ||
vector<string> strVec; | ||
split(strVec, str, is_any_of(",")); | ||
assert(strVec.size()==2); | ||
assert(strVec[0]=="a"); | ||
assert(strVec[1]=="b"); | ||
return 0; | ||
} | ||
EOS | ||
system ENV.cxx, "test.cpp", "-std=c++1y", "-L#{lib}", "-lboost_system", "-o", "test" | ||
system "./test" | ||
end | ||
end |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Are you saying you would like something added to the mongo_db_plugin for 1 and 2 OR that you merely suggest users should enableSharding on there database and collections?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I suggest users should do 1 and 2 on production environment.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can you update the comment to make that clearer.