Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Store budget records in ES #2573

Merged
merged 3 commits into from
Jan 3, 2022
Merged

Store budget records in ES #2573

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8cf4144
Add a missing quotation mark
abitmore Jan 2, 2022
827340a
Store budget records in ES
abitmore Jan 2, 2022
7a30ede
Add tests for storing budget records in ES
abitmore Jan 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion libraries/plugins/elasticsearch/elasticsearch_plugin.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -661,7 +661,7 @@ operation_history_object elasticsearch_plugin::get_operation_by_id(operation_his
"query": {
"match":
{
"account_history.operation_id": )" + operation_id_string + R"("
"account_history.operation_id": ")" + operation_id_string + R"("
}
}
}
Expand Down
12 changes: 11 additions & 1 deletion libraries/plugins/es_objects/es_objects.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
#include <graphene/chain/market_object.hpp>
#include <graphene/chain/asset_object.hpp>
#include <graphene/chain/account_object.hpp>
#include <graphene/chain/budget_record_object.hpp>

#include <graphene/utilities/elasticsearch.hpp>
#include <graphene/utilities/boost_program_options.hpp>
Expand Down Expand Up @@ -83,6 +84,7 @@ class es_objects_plugin_impl
object_options balances { true, false, true, "balance" };
object_options limit_orders { true, false, false, "limitorder" };
object_options asset_bitasset { true, false, true, "bitasset" };
object_options budget { true, false, true, "budget" };

std::string index_prefix = "objects-";
uint32_t start_es_after_block = 0;
Expand Down Expand Up @@ -173,6 +175,7 @@ void es_objects_plugin_impl::sync_db()
loader.load<account_balance_object >( _options.balances );
loader.load<proposal_object >( _options.proposals );
loader.load<limit_order_object >( _options.limit_orders );
loader.load<budget_record_object >( _options.budget );
}

void es_objects_plugin_impl::index_database(const vector<object_id_type>& ids, action_type action)
Expand All @@ -198,7 +201,8 @@ void es_objects_plugin_impl::index_database(const vector<object_id_type>& ids, a
{ asset_id_type::space_type, _options.assets },
{ asset_bitasset_data_id_type::space_type, _options.asset_bitasset },
{ limit_order_id_type::space_type, _options.limit_orders },
{ proposal_id_type::space_type, _options.proposals }
{ proposal_id_type::space_type, _options.proposals },
{ budget_record_id_type::space_type, _options.budget }
};

for( const auto& value: ids )
Expand Down Expand Up @@ -231,6 +235,9 @@ void es_objects_plugin_impl::index_database(const vector<object_id_type>& ids, a
case proposal_id_type::space_type:
prepareTemplate( db.get<proposal_object>(value), opt );
break;
case budget_record_id_type::space_type:
prepareTemplate( db.get<budget_record_object>(value), opt );
break;
default:
break;
}
Expand Down Expand Up @@ -381,6 +388,8 @@ void es_objects_plugin::plugin_set_program_options(
("es-objects-asset-bitasset-store-updates", boost::program_options::value<bool>(),
"Store all updates to the bitasset data (false)")

("es-objects-budget-records", boost::program_options::value<bool>(), "Store budget records (true)")

("es-objects-index-prefix", boost::program_options::value<std::string>(),
"Add a prefix to the index(objects-)")
("es-objects-keep-only-current", boost::program_options::value<bool>(),
Expand Down Expand Up @@ -419,6 +428,7 @@ void detail::es_objects_plugin_impl::plugin_options::init(const boost::program_o
utilities::get_program_option( options, "es-objects-limit-orders-no-delete", limit_orders.no_delete );
utilities::get_program_option( options, "es-objects-asset-bitasset", asset_bitasset.enabled );
utilities::get_program_option( options, "es-objects-asset-bitasset-store-updates", asset_bitasset.store_updates );
utilities::get_program_option( options, "es-objects-budget-records", budget.enabled );
utilities::get_program_option( options, "es-objects-index-prefix", index_prefix );
utilities::get_program_option( options, "es-objects-start-es-after-block", start_es_after_block );
utilities::get_program_option( options, "es-objects-sync-db-on-startup", sync_db_on_startup );
Expand Down
26 changes: 24 additions & 2 deletions tests/elasticsearch/main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,10 @@ BOOST_AUTO_TEST_CASE(elasticsearch_objects) {
es.elasticsearch_url = GRAPHENE_TESTING_ES_URL;
es.index_prefix = es_obj_index_prefix;

// delete all first
// The head block number is 1
BOOST_CHECK_EQUAL( db.head_block_num(), 1u );

// delete all first, this will delete genesis data and data inserted at block 1
auto delete_objects = graphene::utilities::deleteAll(es);
BOOST_REQUIRE(delete_objects); // require successful deletion

Expand Down Expand Up @@ -259,11 +262,30 @@ BOOST_AUTO_TEST_CASE(elasticsearch_objects) {

auto bitasset_data_id = j["hits"]["hits"][size_t(0)]["_source"]["bitasset_data_id"].as_string();
es.endpoint = es.index_prefix + "bitasset/_doc/_search";
es.query = "{ \"query\" : { \"bool\": { \"must\" : [{ \"term\": { \"object_id\": \""+bitasset_data_id+"\"}}] } } }";
es.query = "{ \"query\" : { \"bool\": { \"must\" : [{ \"term\": { \"object_id\": \""
+ bitasset_data_id + "\"}}] } } }";
res = graphene::utilities::simpleQuery(es);
j = fc::json::from_string(res);
auto bitasset_object_id = j["hits"]["hits"][size_t(0)]["_source"]["object_id"].as_string();
BOOST_CHECK_EQUAL(bitasset_object_id, bitasset_data_id);

// maintenance, for budget records
generate_blocks( db.get_dynamic_global_properties().next_maintenance_time );

es.endpoint = es.index_prefix + "budget/_doc/_count";
es.query = "";
fc::wait_for( ES_WAIT_TIME, [&]() {
res = graphene::utilities::getEndPoint(es);
j = fc::json::from_string(res);
if( !j.is_object() )
return false;
const auto& obj = j.get_object();
if( obj.find("count") == obj.end() )
return false;
total = obj["count"].as_string();
return (total == "1"); // new record inserted at the first maintenance block
});

}
}
catch (fc::exception &e) {
Expand Down