Merge pull request #2648 from bitshares/pr-2268-account-history-by-bl…

…ocks

Add option to account history plugin to keep operations that happened in last X blocks in memory

libraries/app/config_util.cpp

@@ -257,7 +257,7 @@ static void create_new_config_file(const fc::path& config_ini_path, const fc::pa
        if( name == "partial-operations" )
           return new_option_description(name, bpo::value<bool>()->default_value(true), o->description() );
        if( name == "max-ops-per-account" )
-          return new_option_description(name, bpo::value<int>()->default_value(100), o->description() );
+          return new_option_description(name, bpo::value<uint64_t>()->default_value(100), o->description() );
        return o;
    };
    graphene::app::detail::deduplicator dedup(modify_option_defaults);

libraries/app/include/graphene/app/plugin.hpp

@@ -135,10 +135,13 @@ namespace impl {
 
 #define DEFAULT_VALUE_VECTOR(value) default_value({fc::json::to_string(value)}, fc::json::to_string(value))
 #define LOAD_VALUE_SET(options, name, container, type) \
-if( options.count(name) > 0 ) { \
+do { \
+   if( options.count(name) > 0 ) { \
       const std::vector<std::string>& ops = options[name].as<std::vector<std::string>>(); \
-      std::transform(ops.begin(), ops.end(), std::inserter(container, container.end()), &graphene::app::impl::dejsonify<type>); \
-}
+      std::transform(ops.begin(), ops.end(), std::inserter(container, container.end()), \
+                     &graphene::app::impl::dejsonify<type>); \
+   } \
+} while (false)
 /// @}
 
 } } //graphene::app

libraries/chain/include/graphene/chain/config.hpp

@@ -32,7 +32,7 @@
 
 #define GRAPHENE_MAX_NESTED_OBJECTS (200)
 
-const std::string GRAPHENE_CURRENT_DB_VERSION = "20220916";
+const std::string GRAPHENE_CURRENT_DB_VERSION = "20220930";
 
 #define GRAPHENE_RECENTLY_MISSED_COUNT_INCREMENT             4
 #define GRAPHENE_RECENTLY_MISSED_COUNT_DECREMENT             3

libraries/plugins/account_history/include/graphene/account_history/account_history_plugin.hpp

@@ -24,17 +24,11 @@
 #pragma once
 
 #include <graphene/app/plugin.hpp>
-#include <graphene/chain/database.hpp>
 
-#include <graphene/chain/operation_history_object.hpp>
-
-#include <fc/thread/future.hpp>
+#include <boost/multi_index/composite_key.hpp>
 
 namespace graphene { namespace account_history {
    using namespace chain;
-   //using namespace graphene::db;
-   //using boost::multi_index_container;
-   //using namespace boost::multi_index;
 
 //
 // Plugins should #define their SPACE_ID's so plugins with
@@ -52,9 +46,40 @@ namespace graphene { namespace account_history {
 
 enum account_history_object_type
 {
-   key_account_object_type = 0
+   exceeded_account_object_type = 0
 };
 
+/// This struct tracks accounts that have exceeded the max-ops-per-account limit
+struct exceeded_account_object : public abstract_object<exceeded_account_object>
+{
+   static constexpr uint8_t space_id = ACCOUNT_HISTORY_SPACE_ID;
+   static constexpr uint8_t type_id  = exceeded_account_object_type;
+
+   /// The ID of the account
+   account_id_type account_id;
+   /// The height of the block containing the oldest (not yet removed) operation related to this account
+   uint32_t        block_num;
+};
+
+struct by_account;
+struct by_block_num;
+using exceeded_account_multi_idx_type = multi_index_container<
+   exceeded_account_object,
+   indexed_by<
+      ordered_unique< tag<by_id>, member< object, object_id_type, &object::id > >,
+      ordered_unique< tag<by_account>,
+         member< exceeded_account_object, account_id_type, &exceeded_account_object::account_id > >,
+      ordered_unique< tag<by_block_num>,
+         composite_key<
+            exceeded_account_object,
+            member< exceeded_account_object, uint32_t, &exceeded_account_object::block_num >,
+            member< object, object_id_type, &object::id >
+         >
+      >
+   >
+>;
+
+using exceeded_account_index = generic_index< exceeded_account_object, exceeded_account_multi_idx_type >;
 
 namespace detail
 {
@@ -81,3 +106,6 @@ class account_history_plugin : public graphene::app::plugin
 };
 
 } } //graphene::account_history
+
+FC_REFLECT_DERIVED( graphene::account_history::exceeded_account_object, (graphene::db::object),
+                    (account_id)(block_num) )

libraries/plugins/witness/witness.cpp

@@ -120,7 +120,7 @@ void witness_plugin::plugin_initialize(const boost::program_options::variables_m
 { try {
    ilog("witness plugin:  plugin_initialize() begin");
    _options = &options;
-   LOAD_VALUE_SET(options, "witness-id", _witnesses, chain::witness_id_type)
+   LOAD_VALUE_SET(options, "witness-id", _witnesses, chain::witness_id_type);
 
    if( options.count("private-key") > 0 )
    {

tests/common/database_fixture.cpp

@@ -207,18 +207,28 @@ std::shared_ptr<boost::program_options::variables_map> database_fixture_base::in
       fc::set_option( options, "p2p-endpoint", std::string( ep ) );
    }
 
+   if (fixture.current_test_name == "min_blocks_to_keep_test")
+   {
+      fc::set_option( options, "partial-operations", true );
+      fc::set_option( options, "max-ops-per-account", (uint64_t)2 );
+      fc::set_option( options, "min-blocks-to-keep", (uint32_t)3 );
+      fc::set_option( options, "max-ops-per-acc-by-min-blocks", (uint64_t)5 );
+   }
    if (fixture.current_test_name == "get_account_history_operations")
    {
       fc::set_option( options, "max-ops-per-account", (uint64_t)75 );
+      fc::set_option( options, "min-blocks-to-keep", (uint32_t)0 );
    }
    if (fixture.current_test_name == "api_limit_get_account_history_operations")
    {
       fc::set_option( options, "max-ops-per-account", (uint64_t)125 );
+      fc::set_option( options, "min-blocks-to-keep", (uint32_t)0 );
       fc::set_option( options, "api-limit-get-account-history-operations", (uint32_t)300 );
    }
    if(fixture.current_test_name =="api_limit_get_account_history")
    {
       fc::set_option( options, "max-ops-per-account", (uint64_t)125 );
+      fc::set_option( options, "min-blocks-to-keep", (uint32_t)0 );
       fc::set_option( options, "api-limit-get-account-history", (uint32_t)250 );
    }
    if(fixture.current_test_name =="api_limit_get_grouped_limit_orders")
@@ -228,6 +238,7 @@ std::shared_ptr<boost::program_options::variables_map> database_fixture_base::in
    if(fixture.current_test_name =="api_limit_get_relative_account_history")
    {
       fc::set_option( options, "max-ops-per-account", (uint64_t)125 );
+      fc::set_option( options, "min-blocks-to-keep", (uint32_t)0 );
       fc::set_option( options, "api-limit-get-relative-account-history", (uint32_t)250 );
    }
    if(fixture.current_test_name =="api_limit_get_account_history_by_operations")