From 9056a2427986b52ab8358d1b0b775d65d2fb9def Mon Sep 17 00:00:00 2001 From: abitmore Date: Mon, 8 Aug 2022 08:51:19 +0000 Subject: [PATCH 1/7] Delete objects from ES before loading from DB --- libraries/plugins/es_objects/es_objects.cpp | 27 ++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp index 0e7994867a..719edb90fb 100644 --- a/libraries/plugins/es_objects/es_objects.cpp +++ b/libraries/plugins/es_objects/es_objects.cpp @@ -110,8 +110,12 @@ class es_objects_plugin_impl { index_database( ids, action_type::deletion ); } void index_database(const vector& ids, action_type action); + /// Load all data from the object database into ES void sync_db(); - void remove_from_database( const object_id_type& id, const plugin_options::object_options& opt ); + /// Delete one object from ES + void delete_from_database( const object_id_type& id, const plugin_options::object_options& opt ); + /// Delete all objects of the specified type from ES + void delete_all_from_database( const plugin_options::object_options& opt ); es_objects_plugin& _self; plugin_options _options; @@ -150,6 +154,10 @@ struct data_loader if( !opt.enabled ) return; + // If no_delete or store_updates is true, do not delete + if( !( opt.no_delete || opt.store_updates ) ) + my->delete_all_from_database( opt ); + db.get_index( ObjType::space_id, ObjType::type_id ).inspect_all_objects( [this, &opt](const graphene::db::object &o) { my->prepareTemplate( static_cast(o), opt ); @@ -213,7 +221,7 @@ void es_objects_plugin_impl::index_database(const vector& ids, a continue; const auto& opt = itr->second; if( action_type::deletion == action ) - remove_from_database( value, opt ); + delete_from_database( value, opt ); else { switch( itr->first ) @@ -247,7 +255,7 @@ void es_objects_plugin_impl::index_database(const vector& ids, a } -void es_objects_plugin_impl::remove_from_database( +void es_objects_plugin_impl::delete_from_database( const object_id_type& id, const es_objects_plugin_impl::plugin_options::object_options& opt ) { if( opt.no_delete ) @@ -266,6 +274,19 @@ void es_objects_plugin_impl::remove_from_database( send_bulk_if_ready(); } +void es_objects_plugin_impl::delete_all_from_database( const plugin_options::object_options& opt ) +{ + if( opt.no_delete ) + return; + // Note: + // 1. The _delete_by_query API deletes the data but keeps the index mapping, so the function is OK. + // Simply deleting the index is probably faster, but it requires the "delete_index" permission, and + // may probably mess up the index mapping and other existing settings. + // Don't know if there is a good way to only delete objects that do not exist in the object database. + // 2. We don't check the return value here, it's probably OK + es->query( _options.index_prefix + opt.index_name + "/_delete_by_query", "{\"query\":{\"match_all\":{}}}" ); +} + template void es_objects_plugin_impl::prepareTemplate( const T& blockchain_object, const es_objects_plugin_impl::plugin_options::object_options& opt ) From adccebfd45f243fecc32629e0618f7b4ef8bd144 Mon Sep 17 00:00:00 2001 From: abitmore Date: Mon, 8 Aug 2022 08:57:57 +0000 Subject: [PATCH 2/7] Remove a unnecessary check --- libraries/plugins/es_objects/es_objects.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp index 719edb90fb..0952a586a5 100644 --- a/libraries/plugins/es_objects/es_objects.cpp +++ b/libraries/plugins/es_objects/es_objects.cpp @@ -276,8 +276,6 @@ void es_objects_plugin_impl::delete_from_database( void es_objects_plugin_impl::delete_all_from_database( const plugin_options::object_options& opt ) { - if( opt.no_delete ) - return; // Note: // 1. The _delete_by_query API deletes the data but keeps the index mapping, so the function is OK. // Simply deleting the index is probably faster, but it requires the "delete_index" permission, and From 1758c796c01adb165c7c03e66b4bdb4be0e4f468 Mon Sep 17 00:00:00 2001 From: abitmore Date: Mon, 8 Aug 2022 09:59:55 +0000 Subject: [PATCH 3/7] Always delete objects from ES first on resync --- libraries/plugins/es_objects/es_objects.cpp | 27 ++++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp index 0952a586a5..ae6b69ccb3 100644 --- a/libraries/plugins/es_objects/es_objects.cpp +++ b/libraries/plugins/es_objects/es_objects.cpp @@ -111,7 +111,7 @@ class es_objects_plugin_impl void index_database(const vector& ids, action_type action); /// Load all data from the object database into ES - void sync_db(); + void sync_db( bool delete_before_load = false ); /// Delete one object from ES void delete_from_database( const object_id_type& id, const plugin_options::object_options& opt ); /// Delete all objects of the specified type from ES @@ -149,13 +149,14 @@ struct data_loader } template - void load( const es_objects_plugin_impl::plugin_options::object_options& opt ) + void load( const es_objects_plugin_impl::plugin_options::object_options& opt, + bool force_delete = false ) { if( !opt.enabled ) return; // If no_delete or store_updates is true, do not delete - if( !( opt.no_delete || opt.store_updates ) ) + if( force_delete || !( opt.no_delete || opt.store_updates ) ) my->delete_all_from_database( opt ); db.get_index( ObjType::space_id, ObjType::type_id ).inspect_all_objects( @@ -165,7 +166,7 @@ struct data_loader } }; -void es_objects_plugin_impl::sync_db() +void es_objects_plugin_impl::sync_db( bool delete_before_load ) { ilog("elasticsearch OBJECTS: loading data from the object database (chain state)"); @@ -176,13 +177,13 @@ void es_objects_plugin_impl::sync_db() data_loader loader( this ); - loader.load( _options.accounts ); - loader.load( _options.assets ); - loader.load( _options.asset_bitasset ); - loader.load( _options.balances ); - loader.load( _options.proposals ); - loader.load( _options.limit_orders ); - loader.load( _options.budget ); + loader.load( _options.accounts, delete_before_load ); + loader.load( _options.assets, delete_before_load ); + loader.load( _options.asset_bitasset, delete_before_load ); + loader.load( _options.balances, delete_before_load ); + loader.load( _options.proposals, delete_before_load ); + loader.load( _options.limit_orders, delete_before_load ); + loader.load( _options.budget, delete_before_load ); } void es_objects_plugin_impl::index_database(const vector& ids, action_type action) @@ -482,7 +483,9 @@ void es_objects_plugin::plugin_initialize(const boost::program_options::variable void es_objects_plugin::plugin_startup() { - if( my->_options.sync_db_on_startup || 0 == database().head_block_num() ) + if( 0 == database().head_block_num() ) + my->sync_db( true ); + else if( my->_options.sync_db_on_startup ) my->sync_db(); } From 0ea89c6a41a1b67069a1fd46b05ae0dbc62d831c Mon Sep 17 00:00:00 2001 From: abitmore Date: Mon, 8 Aug 2022 10:25:28 +0000 Subject: [PATCH 4/7] Add logging --- libraries/plugins/es_objects/es_objects.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp index ae6b69ccb3..50c5de941a 100644 --- a/libraries/plugins/es_objects/es_objects.cpp +++ b/libraries/plugins/es_objects/es_objects.cpp @@ -157,8 +157,12 @@ struct data_loader // If no_delete or store_updates is true, do not delete if( force_delete || !( opt.no_delete || opt.store_updates ) ) + { + ilog( "Deleting all data in index " + my->_options.index_prefix + opt.index_name ); my->delete_all_from_database( opt ); + } + ilog( "Loading data into index " + my->_options.index_prefix + opt.index_name ); db.get_index( ObjType::space_id, ObjType::type_id ).inspect_all_objects( [this, &opt](const graphene::db::object &o) { my->prepareTemplate( static_cast(o), opt ); @@ -184,6 +188,8 @@ void es_objects_plugin_impl::sync_db( bool delete_before_load ) loader.load( _options.proposals, delete_before_load ); loader.load( _options.limit_orders, delete_before_load ); loader.load( _options.budget, delete_before_load ); + + ilog("elasticsearch OBJECTS: done loading data from the object database (chain state)"); } void es_objects_plugin_impl::index_database(const vector& ids, action_type action) From d23a6e1c0c39e62d752c86e95407110f8278ef9f Mon Sep 17 00:00:00 2001 From: abitmore Date: Mon, 8 Aug 2022 10:42:00 +0000 Subject: [PATCH 5/7] Fix code smells --- libraries/plugins/es_objects/es_objects.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp index 50c5de941a..f7a2f4181e 100644 --- a/libraries/plugins/es_objects/es_objects.cpp +++ b/libraries/plugins/es_objects/es_objects.cpp @@ -115,7 +115,7 @@ class es_objects_plugin_impl /// Delete one object from ES void delete_from_database( const object_id_type& id, const plugin_options::object_options& opt ); /// Delete all objects of the specified type from ES - void delete_all_from_database( const plugin_options::object_options& opt ); + void delete_all_from_database( const plugin_options::object_options& opt ) const; es_objects_plugin& _self; plugin_options _options; @@ -281,7 +281,7 @@ void es_objects_plugin_impl::delete_from_database( send_bulk_if_ready(); } -void es_objects_plugin_impl::delete_all_from_database( const plugin_options::object_options& opt ) +void es_objects_plugin_impl::delete_all_from_database( const plugin_options::object_options& opt ) const { // Note: // 1. The _delete_by_query API deletes the data but keeps the index mapping, so the function is OK. @@ -289,7 +289,7 @@ void es_objects_plugin_impl::delete_all_from_database( const plugin_options::obj // may probably mess up the index mapping and other existing settings. // Don't know if there is a good way to only delete objects that do not exist in the object database. // 2. We don't check the return value here, it's probably OK - es->query( _options.index_prefix + opt.index_name + "/_delete_by_query", "{\"query\":{\"match_all\":{}}}" ); + es->query( _options.index_prefix + opt.index_name + "/_delete_by_query", R"({"query":{"match_all":{}}})" ); } template From d54da7ec886550bd7769f122d49c52061741e583 Mon Sep 17 00:00:00 2001 From: abitmore Date: Mon, 8 Aug 2022 11:07:35 +0000 Subject: [PATCH 6/7] Add tests for limit order object processing in ES --- tests/elasticsearch/main.cpp | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/elasticsearch/main.cpp b/tests/elasticsearch/main.cpp index 1e60ed4de3..779c887406 100644 --- a/tests/elasticsearch/main.cpp +++ b/tests/elasticsearch/main.cpp @@ -235,7 +235,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_objects) { if(delete_objects) { // all records deleted // asset and bitasset - create_bitasset("USD", account_id_type()); + asset_id_type usd_id = create_bitasset("USD", account_id_type()).id; generate_block(); string query = "{ \"query\" : { \"bool\" : { \"must\" : [{\"match_all\": {}}] } } }"; @@ -268,8 +268,28 @@ BOOST_AUTO_TEST_CASE(elasticsearch_objects) { auto bitasset_object_id = j["hits"]["hits"][size_t(0)]["_source"]["object_id"].as_string(); BOOST_CHECK_EQUAL(bitasset_object_id, bitasset_data_id); + // create a limit order that expires at the next maintenance time + create_sell_order( account_id_type(), asset(1), asset(1, usd_id), + db.get_dynamic_global_properties().next_maintenance_time ); + generate_block(); + + es.endpoint = es.index_prefix + "limitorder/_doc/_count"; + es.query = ""; + fc::wait_for( ES_WAIT_TIME, [&]() { + res = graphene::utilities::getEndPoint(es); + j = fc::json::from_string(res); + if( !j.is_object() ) + return false; + const auto& obj = j.get_object(); + if( obj.find("count") == obj.end() ) + return false; + total = obj["count"].as_string(); + return (total == "1"); + }); + // maintenance, for budget records generate_blocks( db.get_dynamic_global_properties().next_maintenance_time ); + generate_block(); es.endpoint = es.index_prefix + "budget/_doc/_count"; es.query = ""; @@ -285,6 +305,16 @@ BOOST_AUTO_TEST_CASE(elasticsearch_objects) { return (total == "1"); // new record inserted at the first maintenance block }); + es.endpoint = es.index_prefix + "limitorder/_doc/_count"; + es.query = ""; + res = graphene::utilities::getEndPoint(es); + j = fc::json::from_string(res); + BOOST_REQUIRE( j.is_object() ); + const auto& obj = j.get_object(); + BOOST_REQUIRE( obj.find("count") != obj.end() ); + total = obj["count"].as_string(); + BOOST_CHECK( total == "0" ); // the limit order expired, so the object is removed + } } catch (fc::exception &e) { From d0ead7893cfc794dd3adb4e1ddb46fac74145d5f Mon Sep 17 00:00:00 2001 From: abitmore Date: Mon, 8 Aug 2022 11:57:07 +0000 Subject: [PATCH 7/7] Fix es_objects plugin limit order processing tests --- tests/common/database_fixture.cpp | 4 ++-- tests/elasticsearch/main.cpp | 18 +++++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/common/database_fixture.cpp b/tests/common/database_fixture.cpp index 0240ef86f7..67683e9421 100644 --- a/tests/common/database_fixture.cpp +++ b/tests/common/database_fixture.cpp @@ -413,8 +413,8 @@ std::shared_ptr database_fixture_base::in fixture.app.register_plugin(true); fc::set_option( options, "es-objects-elasticsearch-url", GRAPHENE_TESTING_ES_URL ); - fc::set_option( options, "es-objects-bulk-replay", uint32_t(2) ); - fc::set_option( options, "es-objects-bulk-sync", uint32_t(2) ); + fc::set_option( options, "es-objects-bulk-replay", uint32_t(1) ); + fc::set_option( options, "es-objects-bulk-sync", uint32_t(1) ); fc::set_option( options, "es-objects-proposals", true ); fc::set_option( options, "es-objects-accounts", true ); fc::set_option( options, "es-objects-assets", true ); diff --git a/tests/elasticsearch/main.cpp b/tests/elasticsearch/main.cpp index 779c887406..eb7f5d3795 100644 --- a/tests/elasticsearch/main.cpp +++ b/tests/elasticsearch/main.cpp @@ -307,13 +307,17 @@ BOOST_AUTO_TEST_CASE(elasticsearch_objects) { es.endpoint = es.index_prefix + "limitorder/_doc/_count"; es.query = ""; - res = graphene::utilities::getEndPoint(es); - j = fc::json::from_string(res); - BOOST_REQUIRE( j.is_object() ); - const auto& obj = j.get_object(); - BOOST_REQUIRE( obj.find("count") != obj.end() ); - total = obj["count"].as_string(); - BOOST_CHECK( total == "0" ); // the limit order expired, so the object is removed + fc::wait_for( ES_WAIT_TIME, [&]() { + res = graphene::utilities::getEndPoint(es); + j = fc::json::from_string(res); + if( !j.is_object() ) + return false; + const auto& obj = j.get_object(); + if( obj.find("count") == obj.end() ) + return false; + total = obj["count"].as_string(); + return (total == "0"); // the limit order expired, so the object is removed + }); } }