From 10a382fd0c0a53391ba71023aea0ba41c23caebd Mon Sep 17 00:00:00 2001 From: Alfredo Date: Fri, 19 Apr 2019 21:11:17 -0300 Subject: [PATCH 01/14] serve get_account_history with elasticsearch plugin --- libraries/app/CMakeLists.txt | 2 +- libraries/app/api.cpp | 4 + libraries/app/include/graphene/app/api.hpp | 2 + .../elasticsearch/elasticsearch_plugin.cpp | 152 +++++++- .../elasticsearch/elasticsearch_plugin.hpp | 11 + tests/common/database_fixture.cpp | 9 +- tests/elasticsearch/main.cpp | 324 +++++++++++++++++- 7 files changed, 495 insertions(+), 9 deletions(-) diff --git a/libraries/app/CMakeLists.txt b/libraries/app/CMakeLists.txt index bf4f5c2b00..1d1cb65f9b 100644 --- a/libraries/app/CMakeLists.txt +++ b/libraries/app/CMakeLists.txt @@ -13,7 +13,7 @@ add_library( graphene_app ) # need to link graphene_debug_witness because plugins aren't sufficiently isolated #246 -target_link_libraries( graphene_app graphene_market_history graphene_account_history graphene_grouped_orders graphene_chain fc graphene_db graphene_net graphene_utilities graphene_debug_witness ) +target_link_libraries( graphene_app graphene_market_history graphene_account_history graphene_elasticsearch graphene_grouped_orders graphene_chain fc graphene_db graphene_net graphene_utilities graphene_debug_witness ) target_include_directories( graphene_app PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../egenesis/include" ) diff --git a/libraries/app/api.cpp b/libraries/app/api.cpp index c2631b1293..f1a8b909f7 100644 --- a/libraries/app/api.cpp +++ b/libraries/app/api.cpp @@ -328,6 +328,10 @@ namespace graphene { namespace app { start = node.operation_id; } catch(...) { return result; } + auto es = _app.get_plugin("elasticsearch"); + if(es) + return es->get_account_history(account, stop, limit, start); + const auto& hist_idx = db.get_index_type(); const auto& by_op_idx = hist_idx.indices().get(); auto index_start = by_op_idx.begin(); diff --git a/libraries/app/include/graphene/app/api.hpp b/libraries/app/include/graphene/app/api.hpp index 484cde78c5..14193bcf42 100644 --- a/libraries/app/include/graphene/app/api.hpp +++ b/libraries/app/include/graphene/app/api.hpp @@ -32,6 +32,8 @@ #include +#include + #include #include diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index 0719fc06b4..d2e873b92c 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -26,7 +26,6 @@ #include #include #include -#include namespace graphene { namespace elasticsearch { @@ -59,6 +58,7 @@ class elasticsearch_plugin_impl std::string _elasticsearch_index_prefix = "bitshares-"; bool _elasticsearch_operation_object = false; uint32_t _elasticsearch_start_es_after_block = 0; + bool _elasticsearch_operation_string = true; CURL *curl; // curl handler vector bulk_lines; // vector of op lines vector prepare; @@ -223,9 +223,8 @@ void elasticsearch_plugin_impl::doOperationHistory(const optional op); - } void elasticsearch_plugin_impl::doBlock(uint32_t trx_in_block, const signed_block& b) @@ -436,6 +435,7 @@ void elasticsearch_plugin::plugin_set_program_options( ("elasticsearch-index-prefix", boost::program_options::value(), "Add a prefix to the index(bitshares-)") ("elasticsearch-operation-object", boost::program_options::value(), "Save operation as object(false)") ("elasticsearch-start-es-after-block", boost::program_options::value(), "Start doing ES job after block(0)") + ("elasticsearch-operation-string", boost::program_options::value(), "Save operation as string. Needed to serve history api calls(true)") ; cfg.add(cli); } @@ -473,7 +473,10 @@ void elasticsearch_plugin::plugin_initialize(const boost::program_options::varia } if (options.count("elasticsearch-start-es-after-block")) { my->_elasticsearch_start_es_after_block = options["elasticsearch-start-es-after-block"].as(); - } + } + if (options.count("elasticsearch-operation-string")) { + my->_elasticsearch_operation_string = options["elasticsearch-operation-string"].as(); + } } void elasticsearch_plugin::plugin_startup() @@ -488,4 +491,145 @@ void elasticsearch_plugin::plugin_startup() ilog("elasticsearch ACCOUNT HISTORY: plugin_startup() begin"); } +operation_history_object elasticsearch_plugin::get_operation_by_id(operation_history_id_type id) +{ + const string operation_id_string = idToString(id); + + const string query = R"( + { + "query": { + "bool": { + "must": [ + { + "query_string": { + "query": "account_history.operation_id: )" + operation_id_string + R"(" + } + }, + { + "range": { + "block_data.block_time": { + "gte": "now-20y", + "lte": "now" + } + } + } + ] + } + } + } + )"; + + auto es = prepareHistoryQuery(query); + const auto response = graphene::utilities::simpleQuery(es); + variant variant_response = fc::json::from_string(response); + const auto source = variant_response["hits"]["hits"][size_t(0)]["_source"]; + return fromEStoOperation(source); +} + +vector elasticsearch_plugin::get_account_history( + const account_id_type account_id, + operation_history_id_type stop = operation_history_id_type(), + unsigned limit = 100, + operation_history_id_type start = operation_history_id_type()) +{ + const string account_id_string = idToString(account_id); + + const auto stop_number = stop.instance.value; + const auto start_number = start.instance.value; + + string range = ""; + if(stop_number == 0) + range = " AND operation_id_num: ["+fc::to_string(stop_number)+" TO "+fc::to_string(start_number)+"]"; + else if(stop_number > 0) + range = " AND operation_id_num: {"+fc::to_string(stop_number)+" TO "+fc::to_string(start_number)+"]"; + + const string query = R"( + { + "size": )" + fc::to_string(limit) + R"(, + "sort" : [{ "operation_id_num" : {"order" : "desc"}}], + "query": { + "bool": { + "must": [ + { + "query_string": { + "query": "account_history.account: )" + account_id_string + range + R"(" + } + }, + { + "range": { + "block_data.block_time": { + "gte": "now-20y", + "lte": "now" + } + } + } + ] + } + } + } + )"; + + auto es = prepareHistoryQuery(query); + + vector result; + + if(!graphene::utilities::checkES(es)) + return result; + + const auto response = graphene::utilities::simpleQuery(es); + variant variant_response = fc::json::from_string(response); + + const auto hits = variant_response["hits"]["total"]; + const auto size = std::min(static_cast(hits.as_uint64()), limit); + + for(unsigned i=0; i +std::string elasticsearch_plugin::idToString(T id) +{ + return fc::to_string(id.space_id) + "." + fc::to_string(id.type_id) + "." + fc::to_string(id.instance.value); +} + +graphene::utilities::ES elasticsearch_plugin::prepareHistoryQuery(string query) +{ + CURL *curl; + curl = curl_easy_init(); + + graphene::utilities::ES es; + es.curl = curl; + es.elasticsearch_url = my->_elasticsearch_node_url; + es.index_prefix = my->_elasticsearch_index_prefix; + es.endpoint = es.index_prefix + "*/data/_search"; + es.query = query; + + return es; +} + } } diff --git a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp index a5ee7417c7..d6ec1191aa 100644 --- a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp +++ b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp @@ -26,6 +26,7 @@ #include #include #include +#include namespace graphene { namespace elasticsearch { using namespace chain; @@ -63,8 +64,18 @@ class elasticsearch_plugin : public graphene::app::plugin virtual void plugin_initialize(const boost::program_options::variables_map& options) override; virtual void plugin_startup() override; + operation_history_object get_operation_by_id(operation_history_id_type id); + vector get_account_history(const account_id_type account_id, + operation_history_id_type stop, unsigned limit, operation_history_id_type start); + friend class detail::elasticsearch_plugin_impl; std::unique_ptr my; + + private: + operation_history_object fromEStoOperation(variant source); + template + std::string idToString(T id); + graphene::utilities::ES prepareHistoryQuery(string query); }; struct operation_visitor diff --git a/tests/common/database_fixture.cpp b/tests/common/database_fixture.cpp index af0e9513e5..9bb5691540 100644 --- a/tests/common/database_fixture.cpp +++ b/tests/common/database_fixture.cpp @@ -186,15 +186,18 @@ database_fixture::database_fixture(const fc::time_point_sec &initial_timestamp) boost::unit_test::framework::current_test_case().p_name.value == "track_votes_committee_disabled") { app.chain_database()->enable_standby_votes_tracking( false ); } - if(current_test_name == "elasticsearch_account_history" || current_test_name == "elasticsearch_suite") { + if(current_test_name == "elasticsearch_account_history" || current_test_name == "elasticsearch_suite" || + current_test_name == "elasticsearch_history_api") { auto esplugin = app.register_plugin(); esplugin->plugin_set_app(&app); options.insert(std::make_pair("elasticsearch-node-url", boost::program_options::variable_value(string("http://localhost:9200/"), false))); options.insert(std::make_pair("elasticsearch-bulk-replay", boost::program_options::variable_value(uint32_t(2), false))); options.insert(std::make_pair("elasticsearch-bulk-sync", boost::program_options::variable_value(uint32_t(2), false))); - options.insert(std::make_pair("elasticsearch-visitor", boost::program_options::variable_value(true, false))); - //options.insert(std::make_pair("elasticsearch-basic-auth", boost::program_options::variable_value(string("elastic:changeme"), false))); + options.insert(std::make_pair("elasticsearch-start-es-after-block", boost::program_options::variable_value(uint32_t(0), false))); + options.insert(std::make_pair("elasticsearch-visitor", boost::program_options::variable_value(false, false))); + options.insert(std::make_pair("elasticsearch-operation-object", boost::program_options::variable_value(true, false))); + options.insert(std::make_pair("elasticsearch-operation-string", boost::program_options::variable_value(true, false))); esplugin->plugin_initialize(options); esplugin->plugin_startup(); diff --git a/tests/elasticsearch/main.cpp b/tests/elasticsearch/main.cpp index 33f8b11db9..24eb9382e8 100644 --- a/tests/elasticsearch/main.cpp +++ b/tests/elasticsearch/main.cpp @@ -27,6 +27,7 @@ #include #include +#include #include "../common/database_fixture.hpp" @@ -118,7 +119,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_account_history) { es.endpoint = index_name + "/data/2.9.12"; // we know last op is a transfer of amount 300 res = graphene::utilities::getEndPoint(es); j = fc::json::from_string(res); - auto last_transfer_amount = j["_source"]["additional_data"]["transfer_data"]["amount"].as_string(); + auto last_transfer_amount = j["_source"]["operation_history"]["op_object"]["amount_"]["amount"].as_string(); BOOST_CHECK_EQUAL(last_transfer_amount, "300"); } } @@ -210,4 +211,325 @@ BOOST_AUTO_TEST_CASE(elasticsearch_suite) { } } +BOOST_AUTO_TEST_CASE(elasticsearch_history_api) { + try { + CURL *curl; // curl handler + curl = curl_easy_init(); + + graphene::utilities::ES es; + es.curl = curl; + es.elasticsearch_url = "http://localhost:9200/"; + es.index_prefix = "bitshares-"; + + auto delete_account_history = graphene::utilities::deleteAll(es); + + generate_block(); + fc::usleep(fc::milliseconds(1000)); + + if(delete_account_history) { + + create_bitasset("USD", account_id_type()); // create op 0 + const account_object& dan = create_account("dan"); // create op 1 + create_bitasset("CNY", dan.id); // create op 2 + create_bitasset("BTC", account_id_type()); // create op 3 + create_bitasset("XMR", dan.id); // create op 4 + create_bitasset("EUR", account_id_type()); // create op 5 + create_bitasset("OIL", dan.id); // create op 6 + + generate_block(); + fc::usleep(fc::milliseconds(1000)); + + graphene::app::history_api hist_api(app); + app.enable_plugin("elasticsearch"); + + // f(A, 0, 4, 9) = { 5, 3, 1, 0 } + auto histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(9)); + + BOOST_CHECK_EQUAL(histories.size(), 4u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[3].id.instance(), 0u); + + // f(A, 0, 4, 6) = { 5, 3, 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(6)); + BOOST_CHECK_EQUAL(histories.size(), 4u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[3].id.instance(), 0u); + + // f(A, 0, 4, 5) = { 5, 3, 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(5)); + BOOST_CHECK_EQUAL(histories.size(), 4u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[3].id.instance(), 0u); + + // f(A, 0, 4, 4) = { 3, 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(4)); + BOOST_CHECK_EQUAL(histories.size(), 3u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 0u); + + // f(A, 0, 4, 3) = { 3, 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(3)); + BOOST_CHECK_EQUAL(histories.size(), 3u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 0u); + + // f(A, 0, 4, 2) = { 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(2)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 0u); + + // f(A, 0, 4, 1) = { 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(1)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 0u); + + // f(A, 0, 4, 0) = { 5, 3, 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type()); + BOOST_CHECK_EQUAL(histories.size(), 4u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[3].id.instance(), 0u); + + // f(A, 1, 5, 9) = { 5, 3 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(9)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + + // f(A, 1, 5, 6) = { 5, 3 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(6)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + + // f(A, 1, 5, 5) = { 5, 3 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(5)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + + // f(A, 1, 5, 4) = { 3 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(4)); + BOOST_CHECK_EQUAL(histories.size(), 1u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u); + + // f(A, 1, 5, 3) = { 3 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(3)); + BOOST_CHECK_EQUAL(histories.size(), 1u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u); + + // f(A, 1, 5, 2) = { } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(2)); + BOOST_CHECK_EQUAL(histories.size(), 0u); + + // f(A, 1, 5, 1) = { } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(1)); + BOOST_CHECK_EQUAL(histories.size(), 0u); + + // f(A, 1, 5, 0) = { 5, 3 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(0)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + + // f(A, 0, 3, 9) = { 5, 3, 1 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(9)); + BOOST_CHECK_EQUAL(histories.size(), 3u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u); + + // f(A, 0, 3, 6) = { 5, 3, 1 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(6)); + BOOST_CHECK_EQUAL(histories.size(), 3u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u); + + // f(A, 0, 3, 5) = { 5, 3, 1 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(5)); + BOOST_CHECK_EQUAL(histories.size(), 3u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u); + + // f(A, 0, 3, 4) = { 3, 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(4)); + BOOST_CHECK_EQUAL(histories.size(), 3u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 0u); + + // f(A, 0, 3, 3) = { 3, 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(3)); + BOOST_CHECK_EQUAL(histories.size(), 3u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 0u); + + // f(A, 0, 3, 2) = { 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(2)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 0u); + + // f(A, 0, 3, 1) = { 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(1)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 0u); + + // f(A, 0, 3, 0) = { 5, 3, 1 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type()); + BOOST_CHECK_EQUAL(histories.size(), 3u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u); + + // f(B, 0, 4, 9) = { 6, 4, 2, 1 } + histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(9)); + BOOST_CHECK_EQUAL(histories.size(), 4u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 2u); + BOOST_CHECK_EQUAL(histories[3].id.instance(), 1u); + + // f(B, 0, 4, 6) = { 6, 4, 2, 1 } + histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(6)); + BOOST_CHECK_EQUAL(histories.size(), 4u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 2u); + BOOST_CHECK_EQUAL(histories[3].id.instance(), 1u); + + // f(B, 0, 4, 5) = { 4, 2, 1 } + histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(5)); + BOOST_CHECK_EQUAL(histories.size(), 3u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 4u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 2u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u); + + // f(B, 0, 4, 4) = { 4, 2, 1 } + histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(4)); + BOOST_CHECK_EQUAL(histories.size(), 3u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 4u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 2u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u); + + // f(B, 0, 4, 3) = { 2, 1 } + histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(3)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 2u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u); + + // f(B, 0, 4, 2) = { 2, 1 } + histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(2)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 2u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u); + + // f(B, 0, 4, 1) = { 1 } + histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(1)); + BOOST_CHECK_EQUAL(histories.size(), 1u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 1u); + + // f(B, 0, 4, 0) = { 6, 4, 2, 1 } + histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type()); + BOOST_CHECK_EQUAL(histories.size(), 4u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 2u); + BOOST_CHECK_EQUAL(histories[3].id.instance(), 1u); + + // f(B, 2, 4, 9) = { 6, 4 } + histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(9)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u); + + // f(B, 2, 4, 6) = { 6, 4 } + histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(6)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u); + + // f(B, 2, 4, 5) = { 4 } + histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(5)); + BOOST_CHECK_EQUAL(histories.size(), 1u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 4u); + + // f(B, 2, 4, 4) = { 4 } + histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(4)); + BOOST_CHECK_EQUAL(histories.size(), 1u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 4u); + + // f(B, 2, 4, 3) = { } + histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(3)); + BOOST_CHECK_EQUAL(histories.size(), 0u); + + // f(B, 2, 4, 2) = { } + histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(2)); + BOOST_CHECK_EQUAL(histories.size(), 0u); + + // f(B, 2, 4, 1) = { } + histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(1)); + BOOST_CHECK_EQUAL(histories.size(), 0u); + + // f(B, 2, 4, 0) = { 6, 4 } + histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(0)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u); + + // 0 limits + histories = hist_api.get_account_history("dan", operation_history_id_type(0), 0, operation_history_id_type(0)); + BOOST_CHECK_EQUAL(histories.size(), 0u); + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(3), 0, operation_history_id_type(9)); + BOOST_CHECK_EQUAL(histories.size(), 0u); + + // non existent account + histories = hist_api.get_account_history("1.2.18", operation_history_id_type(0), 4, operation_history_id_type(0)); + BOOST_CHECK_EQUAL(histories.size(), 0u); + + // create a new account C = alice { 7 } + create_account("alice"); + + generate_block(); + fc::usleep(fc::milliseconds(1000)); + + // f(C, 0, 4, 10) = { 7 } + histories = hist_api.get_account_history("alice", operation_history_id_type(0), 4, operation_history_id_type(10)); + BOOST_CHECK_EQUAL(histories.size(), 1u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 7u); + + // f(C, 8, 4, 10) = { } + histories = hist_api.get_account_history("alice", operation_history_id_type(8), 4, operation_history_id_type(10)); + BOOST_CHECK_EQUAL(histories.size(), 0u); + + // f(A, 0, 10, 0) = { 7, 5, 3, 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(0), 10, operation_history_id_type(0)); + BOOST_CHECK_EQUAL(histories.size(), 5u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 7u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[3].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[4].id.instance(), 0u); + } + } + catch (fc::exception &e) { + edump((e.to_detail_string())); + throw; + } +} BOOST_AUTO_TEST_SUITE_END() From 00496a96b4abf10c37704c9dff95e22500784ad9 Mon Sep 17 00:00:00 2001 From: Alfredo Date: Sat, 20 Apr 2019 10:32:12 -0300 Subject: [PATCH 02/14] replace IdToString --- .../plugins/elasticsearch/elasticsearch_plugin.cpp | 10 ++-------- .../graphene/elasticsearch/elasticsearch_plugin.hpp | 2 -- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index d2e873b92c..9f3e70599a 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -493,7 +493,7 @@ void elasticsearch_plugin::plugin_startup() operation_history_object elasticsearch_plugin::get_operation_by_id(operation_history_id_type id) { - const string operation_id_string = idToString(id); + const string operation_id_string = std::string(object_id_type(id)); const string query = R"( { @@ -532,7 +532,7 @@ vector elasticsearch_plugin::get_account_history( unsigned limit = 100, operation_history_id_type start = operation_history_id_type()) { - const string account_id_string = idToString(account_id); + const string account_id_string = std::string(object_id_type(account_id)); const auto stop_number = stop.instance.value; const auto start_number = start.instance.value; @@ -611,12 +611,6 @@ operation_history_object elasticsearch_plugin::fromEStoOperation(variant source) return result; } -template -std::string elasticsearch_plugin::idToString(T id) -{ - return fc::to_string(id.space_id) + "." + fc::to_string(id.type_id) + "." + fc::to_string(id.instance.value); -} - graphene::utilities::ES elasticsearch_plugin::prepareHistoryQuery(string query) { CURL *curl; diff --git a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp index d6ec1191aa..7cff0b73b8 100644 --- a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp +++ b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp @@ -73,8 +73,6 @@ class elasticsearch_plugin : public graphene::app::plugin private: operation_history_object fromEStoOperation(variant source); - template - std::string idToString(T id); graphene::utilities::ES prepareHistoryQuery(string query); }; From 2a6d7b26c861dfbe691d8e1233792c121a0eee8e Mon Sep 17 00:00:00 2001 From: Alfredo Date: Mon, 15 Jul 2019 13:45:12 -0300 Subject: [PATCH 03/14] add is_plugin_enabled to application --- libraries/app/api.cpp | 5 +++-- libraries/app/application.cpp | 7 +++++++ libraries/app/include/graphene/app/application.hpp | 5 ++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/libraries/app/api.cpp b/libraries/app/api.cpp index f1a8b909f7..2bdc8de93e 100644 --- a/libraries/app/api.cpp +++ b/libraries/app/api.cpp @@ -328,9 +328,10 @@ namespace graphene { namespace app { start = node.operation_id; } catch(...) { return result; } - auto es = _app.get_plugin("elasticsearch"); - if(es) + if(_app.is_plugin_enabled("elasticsearch")) { + auto es = _app.get_plugin("elasticsearch"); return es->get_account_history(account, stop, limit, start); + } const auto& hist_idx = db.get_index_type(); const auto& by_op_idx = hist_idx.indices().get(); diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp index 4217b2e934..909bfd87e1 100644 --- a/libraries/app/application.cpp +++ b/libraries/app/application.cpp @@ -1073,6 +1073,13 @@ std::shared_ptr application::get_plugin(const string& name) con return my->_active_plugins[name]; } +bool application::is_plugin_enabled(const string& name) const +{ + if(my->_active_plugins.find(name) == my->_active_plugins.end()) + return false; + return true; +} + net::node_ptr application::p2p_node() { return my->_p2p_network; diff --git a/libraries/app/include/graphene/app/application.hpp b/libraries/app/include/graphene/app/application.hpp index 6f1a0d6e90..c1feb73422 100644 --- a/libraries/app/include/graphene/app/application.hpp +++ b/libraries/app/include/graphene/app/application.hpp @@ -117,7 +117,10 @@ namespace graphene { namespace app { void enable_plugin( const string& name ); - private: + bool is_plugin_enabled(const string& name) const; + + + private: void add_available_plugin( std::shared_ptr p ); std::shared_ptr my; From 1737d455e9c087159287133f55f4a879b9c0d913 Mon Sep 17 00:00:00 2001 From: Alfredo Date: Mon, 15 Jul 2019 18:40:52 -0300 Subject: [PATCH 04/14] add elasticsearch-mode command for save, query or all --- .../elasticsearch/elasticsearch_plugin.cpp | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index 9f3e70599a..c68454a153 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -59,6 +59,7 @@ class elasticsearch_plugin_impl bool _elasticsearch_operation_object = false; uint32_t _elasticsearch_start_es_after_block = 0; bool _elasticsearch_operation_string = true; + std::string _elasticsearch_mode = "only_save"; CURL *curl; // curl handler vector bulk_lines; // vector of op lines vector prepare; @@ -436,17 +437,13 @@ void elasticsearch_plugin::plugin_set_program_options( ("elasticsearch-operation-object", boost::program_options::value(), "Save operation as object(false)") ("elasticsearch-start-es-after-block", boost::program_options::value(), "Start doing ES job after block(0)") ("elasticsearch-operation-string", boost::program_options::value(), "Save operation as string. Needed to serve history api calls(true)") + ("elasticsearch-mode", boost::program_options::value(), "Mode of operation: only_save, only_query, all(only_save)") ; cfg.add(cli); } void elasticsearch_plugin::plugin_initialize(const boost::program_options::variables_map& options) { - database().applied_block.connect( [&]( const signed_block& b) { - if (!my->update_account_histories(b)) - FC_THROW_EXCEPTION(graphene::chain::plugin_exception, "Error populating ES database, we are going to keep trying."); - } ); - my->_oho_index = database().add_index< primary_index< operation_history_index > >(); database().add_index< primary_index< account_transaction_history_index > >(); @@ -477,6 +474,19 @@ void elasticsearch_plugin::plugin_initialize(const boost::program_options::varia if (options.count("elasticsearch-operation-string")) { my->_elasticsearch_operation_string = options["elasticsearch-operation-string"].as(); } + if (options.count("elasticsearch-mode")) { + my->_elasticsearch_mode = options["elasticsearch-mode"].as(); + } + + if(my->_elasticsearch_mode != "only_query") { + if (my->_elasticsearch_mode == "all") + my->_elasticsearch_operation_string = true; + + database().applied_block.connect([&](const signed_block &b) { + if (!my->update_account_histories(b)) + FC_THROW_EXCEPTION(graphene::chain::plugin_exception, "Error populating ES database, we are going to keep trying."); + }); + } } void elasticsearch_plugin::plugin_startup() From 6dda9fe28c9418570a749a83f09e4bb7e5325cd1 Mon Sep 17 00:00:00 2001 From: Alfredo Date: Tue, 16 Jul 2019 11:52:28 -0300 Subject: [PATCH 05/14] execute elasticsearch account_history in a new thread --- libraries/app/api.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libraries/app/api.cpp b/libraries/app/api.cpp index 2bdc8de93e..7b3d36a02f 100644 --- a/libraries/app/api.cpp +++ b/libraries/app/api.cpp @@ -330,7 +330,9 @@ namespace graphene { namespace app { if(_app.is_plugin_enabled("elasticsearch")) { auto es = _app.get_plugin("elasticsearch"); - return es->get_account_history(account, stop, limit, start); + auto _thread = std::make_shared("elasticsearch"); + return _thread->async([&](){ return es->get_account_history(account, stop, limit, start); }, + "thread invoke for method " BOOST_PP_STRINGIZE(method_name)).wait(); } const auto& hist_idx = db.get_index_type(); From 26e9f16b3f5f88cd186ec3c853aa7278d73c7419 Mon Sep 17 00:00:00 2001 From: Alfredo Date: Wed, 31 Jul 2019 10:16:36 -0300 Subject: [PATCH 06/14] simplify conditional --- libraries/app/application.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp index 909bfd87e1..47e4cbc350 100644 --- a/libraries/app/application.cpp +++ b/libraries/app/application.cpp @@ -1075,9 +1075,7 @@ std::shared_ptr application::get_plugin(const string& name) con bool application::is_plugin_enabled(const string& name) const { - if(my->_active_plugins.find(name) == my->_active_plugins.end()) - return false; - return true; + return !(my->_active_plugins.find(name) == my->_active_plugins.end()); } net::node_ptr application::p2p_node() From 4fac171544b4a8f7859fc0ed56c06eddb3e6ff92 Mon Sep 17 00:00:00 2001 From: Alfredo Date: Wed, 31 Jul 2019 10:42:18 -0300 Subject: [PATCH 07/14] capture only what is necessary in lambda --- libraries/plugins/elasticsearch/elasticsearch_plugin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index c68454a153..1069b34269 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -482,7 +482,7 @@ void elasticsearch_plugin::plugin_initialize(const boost::program_options::varia if (my->_elasticsearch_mode == "all") my->_elasticsearch_operation_string = true; - database().applied_block.connect([&](const signed_block &b) { + database().applied_block.connect([this](const signed_block &b) { if (!my->update_account_histories(b)) FC_THROW_EXCEPTION(graphene::chain::plugin_exception, "Error populating ES database, we are going to keep trying."); }); From 29bf9758f18236ff42d9dfda4df546c96c14e9fd Mon Sep 17 00:00:00 2001 From: Alfredo Date: Wed, 31 Jul 2019 10:54:34 -0300 Subject: [PATCH 08/14] wrap command line option long lines --- .../elasticsearch/elasticsearch_plugin.cpp | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index 1069b34269..98fc06d997 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -428,16 +428,26 @@ void elasticsearch_plugin::plugin_set_program_options( ) { cli.add_options() - ("elasticsearch-node-url", boost::program_options::value(), "Elastic Search database node url(http://localhost:9200/)") - ("elasticsearch-bulk-replay", boost::program_options::value(), "Number of bulk documents to index on replay(10000)") - ("elasticsearch-bulk-sync", boost::program_options::value(), "Number of bulk documents to index on a syncronied chain(100)") - ("elasticsearch-visitor", boost::program_options::value(), "Use visitor to index additional data(slows down the replay(false))") - ("elasticsearch-basic-auth", boost::program_options::value(), "Pass basic auth to elasticsearch database('')") - ("elasticsearch-index-prefix", boost::program_options::value(), "Add a prefix to the index(bitshares-)") - ("elasticsearch-operation-object", boost::program_options::value(), "Save operation as object(false)") - ("elasticsearch-start-es-after-block", boost::program_options::value(), "Start doing ES job after block(0)") - ("elasticsearch-operation-string", boost::program_options::value(), "Save operation as string. Needed to serve history api calls(true)") - ("elasticsearch-mode", boost::program_options::value(), "Mode of operation: only_save, only_query, all(only_save)") + ("elasticsearch-node-url", boost::program_options::value(), + "Elastic Search database node url(http://localhost:9200/)") + ("elasticsearch-bulk-replay", boost::program_options::value(), + "Number of bulk documents to index on replay(10000)") + ("elasticsearch-bulk-sync", boost::program_options::value(), + "Number of bulk documents to index on a syncronied chain(100)") + ("elasticsearch-visitor", boost::program_options::value(), + "Use visitor to index additional data(slows down the replay(false))") + ("elasticsearch-basic-auth", boost::program_options::value(), + "Pass basic auth to elasticsearch database('')") + ("elasticsearch-index-prefix", boost::program_options::value(), + "Add a prefix to the index(bitshares-)") + ("elasticsearch-operation-object", boost::program_options::value(), + "Save operation as object(false)") + ("elasticsearch-start-es-after-block", boost::program_options::value(), + "Start doing ES job after block(0)") + ("elasticsearch-operation-string", boost::program_options::value(), + "Save operation as string. Needed to serve history api calls(true)") + ("elasticsearch-mode", boost::program_options::value(), + "Mode of operation: only_save, only_query, all(only_save)") ; cfg.add(cli); } From de76301d536539675b90357720a758d849af3768 Mon Sep 17 00:00:00 2001 From: Alfredo Date: Wed, 31 Jul 2019 16:02:58 -0300 Subject: [PATCH 09/14] make mode an enum for comparison performance --- .../elasticsearch/elasticsearch_plugin.cpp | 15 +++++++++------ .../elasticsearch/elasticsearch_plugin.hpp | 3 +++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index 98fc06d997..cbdec6fa93 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -59,7 +59,7 @@ class elasticsearch_plugin_impl bool _elasticsearch_operation_object = false; uint32_t _elasticsearch_start_es_after_block = 0; bool _elasticsearch_operation_string = true; - std::string _elasticsearch_mode = "only_save"; + mode _elasticsearch_mode = mode::only_save; CURL *curl; // curl handler vector bulk_lines; // vector of op lines vector prepare; @@ -446,8 +446,8 @@ void elasticsearch_plugin::plugin_set_program_options( "Start doing ES job after block(0)") ("elasticsearch-operation-string", boost::program_options::value(), "Save operation as string. Needed to serve history api calls(true)") - ("elasticsearch-mode", boost::program_options::value(), - "Mode of operation: only_save, only_query, all(only_save)") + ("elasticsearch-mode", boost::program_options::value(), + "Mode of operation: only_save(0), only_query(1), all(2) - Default: 0") ; cfg.add(cli); } @@ -485,11 +485,14 @@ void elasticsearch_plugin::plugin_initialize(const boost::program_options::varia my->_elasticsearch_operation_string = options["elasticsearch-operation-string"].as(); } if (options.count("elasticsearch-mode")) { - my->_elasticsearch_mode = options["elasticsearch-mode"].as(); + const auto option_number = options["elasticsearch-mode"].as(); + if(option_number > mode::all) + FC_THROW_EXCEPTION(fc::exception, "Elasticsearch mode not valid"); + my->_elasticsearch_mode = static_cast(options["elasticsearch-mode"].as()); } - if(my->_elasticsearch_mode != "only_query") { - if (my->_elasticsearch_mode == "all") + if(my->_elasticsearch_mode != mode::only_query) { + if (my->_elasticsearch_mode == mode::all) my->_elasticsearch_operation_string = true; database().applied_block.connect([this](const signed_block &b) { diff --git a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp index 7cff0b73b8..eefbdf05f1 100644 --- a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp +++ b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp @@ -76,6 +76,8 @@ class elasticsearch_plugin : public graphene::app::plugin graphene::utilities::ES prepareHistoryQuery(string query); }; +enum mode { only_save = 0 , only_query = 1, all = 2 }; + struct operation_visitor { typedef void result_type; @@ -302,6 +304,7 @@ struct adaptor_struct { } } //graphene::elasticsearch +FC_REFLECT_ENUM( graphene::elasticsearch::mode, (only_save)(only_query)(all) ) FC_REFLECT( graphene::elasticsearch::operation_history_struct, (trx_in_block)(op_in_trx)(operation_result)(virtual_op)(op)(op_object) ) FC_REFLECT( graphene::elasticsearch::block_struct, (block_num)(block_time)(trx_id) ) FC_REFLECT( graphene::elasticsearch::fee_struct, (asset)(asset_name)(amount)(amount_units) ) From 5838a38fbc93b2673e7c96640abe15a5caa4533e Mon Sep 17 00:00:00 2001 From: Alfredo Date: Wed, 31 Jul 2019 16:59:50 -0300 Subject: [PATCH 10/14] throw when mode::all if elasticsearch-operation-string is false --- .../plugins/elasticsearch/elasticsearch_plugin.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index cbdec6fa93..3463b91ae4 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -487,17 +487,19 @@ void elasticsearch_plugin::plugin_initialize(const boost::program_options::varia if (options.count("elasticsearch-mode")) { const auto option_number = options["elasticsearch-mode"].as(); if(option_number > mode::all) - FC_THROW_EXCEPTION(fc::exception, "Elasticsearch mode not valid"); + FC_THROW_EXCEPTION(graphene::chain::plugin_exception, "Elasticsearch mode not valid"); my->_elasticsearch_mode = static_cast(options["elasticsearch-mode"].as()); } if(my->_elasticsearch_mode != mode::only_query) { - if (my->_elasticsearch_mode == mode::all) - my->_elasticsearch_operation_string = true; + if (my->_elasticsearch_mode == mode::all && !my->_elasticsearch_operation_string) + FC_THROW_EXCEPTION(graphene::chain::plugin_exception, + "If elasticsearch-mode is set to all then elasticsearch-operation-string need to be true"); database().applied_block.connect([this](const signed_block &b) { if (!my->update_account_histories(b)) - FC_THROW_EXCEPTION(graphene::chain::plugin_exception, "Error populating ES database, we are going to keep trying."); + FC_THROW_EXCEPTION(graphene::chain::plugin_exception, + "Error populating ES database, we are going to keep trying."); }); } } From 1f6ac147d5b298fecf17e7bc281676160192ea70 Mon Sep 17 00:00:00 2001 From: Alfredo Date: Wed, 31 Jul 2019 17:33:37 -0300 Subject: [PATCH 11/14] change ES query --- .../elasticsearch/elasticsearch_plugin.cpp | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index 3463b91ae4..f48d2e6f82 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -523,22 +523,9 @@ operation_history_object elasticsearch_plugin::get_operation_by_id(operation_his const string query = R"( { "query": { - "bool": { - "must": [ - { - "query_string": { - "query": "account_history.operation_id: )" + operation_id_string + R"(" - } - }, - { - "range": { - "block_data.block_time": { - "gte": "now-20y", - "lte": "now" - } - } - } - ] + "match": + { + "account_history.operation_id": )" + operation_id_string + R"(" } } } From 63f7affc81b0ca8cb274c97f2124d2d0d8d7ec05 Mon Sep 17 00:00:00 2001 From: Alfredo Date: Wed, 31 Jul 2019 18:25:42 -0300 Subject: [PATCH 12/14] remove range from second ES query --- libraries/plugins/elasticsearch/elasticsearch_plugin.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index f48d2e6f82..fa82b9ba6f 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -566,14 +566,6 @@ vector elasticsearch_plugin::get_account_history( "query_string": { "query": "account_history.account: )" + account_id_string + range + R"(" } - }, - { - "range": { - "block_data.block_time": { - "gte": "now-20y", - "lte": "now" - } - } } ] } From a7fc6b0e9a4bfb379d2a04bbf0e6c4ed026325e6 Mon Sep 17 00:00:00 2001 From: Alfredo Date: Wed, 31 Jul 2019 19:43:01 -0300 Subject: [PATCH 13/14] add get_running_mode() to plugin and use it in api, fix capture --- libraries/app/api.cpp | 9 ++++++--- libraries/plugins/elasticsearch/elasticsearch_plugin.cpp | 6 ++++++ .../graphene/elasticsearch/elasticsearch_plugin.hpp | 4 +++- tests/common/database_fixture.cpp | 1 + 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/libraries/app/api.cpp b/libraries/app/api.cpp index 7b3d36a02f..68400016dd 100644 --- a/libraries/app/api.cpp +++ b/libraries/app/api.cpp @@ -330,9 +330,12 @@ namespace graphene { namespace app { if(_app.is_plugin_enabled("elasticsearch")) { auto es = _app.get_plugin("elasticsearch"); - auto _thread = std::make_shared("elasticsearch"); - return _thread->async([&](){ return es->get_account_history(account, stop, limit, start); }, - "thread invoke for method " BOOST_PP_STRINGIZE(method_name)).wait(); + if(es.get()->get_running_mode() != elasticsearch::mode::only_save) { + auto _thread = std::make_shared("elasticsearch"); + return _thread->async([&es, &account, &stop, &limit, &start]() { + return es->get_account_history(account, stop, limit, start); + }, "thread invoke for method " BOOST_PP_STRINGIZE(method_name)).wait(); + } } const auto& hist_idx = db.get_index_type(); diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index fa82b9ba6f..5729035ee6 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -630,4 +630,10 @@ graphene::utilities::ES elasticsearch_plugin::prepareHistoryQuery(string query) return es; } +mode elasticsearch_plugin::get_running_mode() +{ + return my->_elasticsearch_mode; +} + + } } diff --git a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp index eefbdf05f1..7b08e1d73f 100644 --- a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp +++ b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp @@ -50,6 +50,8 @@ namespace detail class elasticsearch_plugin_impl; } +enum mode { only_save = 0 , only_query = 1, all = 2 }; + class elasticsearch_plugin : public graphene::app::plugin { public: @@ -67,6 +69,7 @@ class elasticsearch_plugin : public graphene::app::plugin operation_history_object get_operation_by_id(operation_history_id_type id); vector get_account_history(const account_id_type account_id, operation_history_id_type stop, unsigned limit, operation_history_id_type start); + mode get_running_mode(); friend class detail::elasticsearch_plugin_impl; std::unique_ptr my; @@ -76,7 +79,6 @@ class elasticsearch_plugin : public graphene::app::plugin graphene::utilities::ES prepareHistoryQuery(string query); }; -enum mode { only_save = 0 , only_query = 1, all = 2 }; struct operation_visitor { diff --git a/tests/common/database_fixture.cpp b/tests/common/database_fixture.cpp index 9bb5691540..cd563828df 100644 --- a/tests/common/database_fixture.cpp +++ b/tests/common/database_fixture.cpp @@ -198,6 +198,7 @@ database_fixture::database_fixture(const fc::time_point_sec &initial_timestamp) options.insert(std::make_pair("elasticsearch-visitor", boost::program_options::variable_value(false, false))); options.insert(std::make_pair("elasticsearch-operation-object", boost::program_options::variable_value(true, false))); options.insert(std::make_pair("elasticsearch-operation-string", boost::program_options::variable_value(true, false))); + options.insert(std::make_pair("elasticsearch-mode", boost::program_options::variable_value(uint16_t(2), false))); esplugin->plugin_initialize(options); esplugin->plugin_startup(); From 07de45e51e2f0ebf35fe24cdcd734a4e54d33500 Mon Sep 17 00:00:00 2001 From: Alfredo Date: Thu, 1 Aug 2019 14:49:57 -0300 Subject: [PATCH 14/14] create only 1 thread to serve ES queries --- libraries/app/api.cpp | 6 ++++-- libraries/app/include/graphene/app/application.hpp | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/libraries/app/api.cpp b/libraries/app/api.cpp index 68400016dd..753b98f413 100644 --- a/libraries/app/api.cpp +++ b/libraries/app/api.cpp @@ -331,8 +331,10 @@ namespace graphene { namespace app { if(_app.is_plugin_enabled("elasticsearch")) { auto es = _app.get_plugin("elasticsearch"); if(es.get()->get_running_mode() != elasticsearch::mode::only_save) { - auto _thread = std::make_shared("elasticsearch"); - return _thread->async([&es, &account, &stop, &limit, &start]() { + if(!_app.elasticsearch_thread) + _app.elasticsearch_thread= std::make_shared("elasticsearch"); + + return _app.elasticsearch_thread->async([&es, &account, &stop, &limit, &start]() { return es->get_account_history(account, stop, limit, start); }, "thread invoke for method " BOOST_PP_STRINGIZE(method_name)).wait(); } diff --git a/libraries/app/include/graphene/app/application.hpp b/libraries/app/include/graphene/app/application.hpp index c1feb73422..2f2a5d4393 100644 --- a/libraries/app/include/graphene/app/application.hpp +++ b/libraries/app/include/graphene/app/application.hpp @@ -119,6 +119,7 @@ namespace graphene { namespace app { bool is_plugin_enabled(const string& name) const; + std::shared_ptr elasticsearch_thread; private: void add_available_plugin( std::shared_ptr p );