diff --git a/nano/lib/numbers.cpp b/nano/lib/numbers.cpp index b546c0cd62..bffd37e396 100644 --- a/nano/lib/numbers.cpp +++ b/nano/lib/numbers.cpp @@ -749,7 +749,7 @@ std::string format_balance (nano::uint128_t balance, nano::uint128_t scale, int return stream.str (); } -std::string nano::uint128_union::format_balance (nano::uint128_t scale, int precision, bool group_digits) +std::string nano::uint128_union::format_balance (nano::uint128_t scale, int precision, bool group_digits) const { auto thousands_sep = std::use_facet> (std::locale ()).thousands_sep (); auto decimal_point = std::use_facet> (std::locale ()).decimal_point (); @@ -757,7 +757,7 @@ std::string nano::uint128_union::format_balance (nano::uint128_t scale, int prec return ::format_balance (number (), scale, precision, group_digits, thousands_sep, decimal_point, grouping); } -std::string nano::uint128_union::format_balance (nano::uint128_t scale, int precision, bool group_digits, const std::locale & locale) +std::string nano::uint128_union::format_balance (nano::uint128_t scale, int precision, bool group_digits, const std::locale & locale) const { auto thousands_sep = std::use_facet> (locale).thousands_sep (); auto decimal_point = std::use_facet> (locale).decimal_point (); diff --git a/nano/lib/numbers.hpp b/nano/lib/numbers.hpp index fece757bc3..01f68744d4 100644 --- a/nano/lib/numbers.hpp +++ b/nano/lib/numbers.hpp @@ -34,8 +34,8 @@ class uint128_union void encode_dec (std::string &) const; bool decode_dec (std::string const &, bool = false); bool decode_dec (std::string const &, nano::uint128_t); - std::string format_balance (nano::uint128_t scale, int precision, bool group_digits); - std::string format_balance (nano::uint128_t scale, int precision, bool group_digits, const std::locale & locale); + std::string format_balance (nano::uint128_t scale, int precision, bool group_digits) const; + std::string format_balance (nano::uint128_t scale, int precision, bool group_digits, const std::locale & locale) const; nano::uint128_t number () const; void clear (); bool is_zero () const; diff --git a/nano/nano_node/entry.cpp b/nano/nano_node/entry.cpp index f14e3f00df..52504d6b1e 100644 --- a/nano/nano_node/entry.cpp +++ b/nano/nano_node/entry.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -69,6 +70,7 @@ int main (int argc, char * const * argv) ("version", "Prints out version") ("config", boost::program_options::value>()->multitoken(), "Pass node configuration values. This takes precedence over any values in the configuration file. This option can be repeated multiple times.") ("daemon", "Start node daemon") + ("compare_rep_weights", "Display a summarized comparison between the hardcoded bootstrap weights and representative weights from the ledger. Full comparison is output to logs") ("debug_block_count", "Display the number of block") ("debug_bootstrap_generate", "Generate bootstrap sequence of blocks") ("debug_dump_frontier_unchecked_dependents", "Dump frontiers which have matching unchecked keys") @@ -158,6 +160,151 @@ int main (int argc, char * const * argv) } daemon.run (data_path, flags); } + else if (vm.count ("compare_rep_weights")) + { + if (!nano::network_constants ().is_test_network ()) + { + auto node_flags = nano::inactive_node_flag_defaults (); + nano::update_flags (node_flags, vm); + node_flags.generate_cache.reps = true; + auto inactive_node = nano::default_inactive_node (data_path, vm); + auto node = inactive_node->node; + + auto const hardcoded = node->get_bootstrap_weights ().second; + auto const ledger_unfiltered = node->ledger.cache.rep_weights.get_rep_amounts (); + + auto get_total = [](decltype (hardcoded) const & reps) -> nano::uint128_union { + return std::accumulate (reps.begin (), reps.end (), nano::uint128_t{ 0 }, [](auto sum, auto const & rep) { return sum + rep.second; }); + }; + + // Hardcoded weights are filtered to a cummulative weight of 99%, need to do the same for ledger weights + std::remove_const_t ledger; + { + std::vector> sorted; + sorted.reserve (ledger_unfiltered.size ()); + std::copy (ledger_unfiltered.begin (), ledger_unfiltered.end (), std::back_inserter (sorted)); + std::sort (sorted.begin (), sorted.end (), [](auto const & left, auto const & right) { return left.second > right.second; }); + auto const total_unfiltered = get_total (ledger_unfiltered); + nano::uint128_t sum{ 0 }; + auto target = (total_unfiltered.number () / 100) * 99; + for (auto i (sorted.begin ()), n (sorted.end ()); i != n && sum <= target; sum += i->second, ++i) + { + ledger.insert (*i); + } + } + + auto const total_ledger = get_total (ledger); + auto const total_hardcoded = get_total (hardcoded); + + struct mismatched_t + { + nano::account rep; + nano::uint128_union hardcoded; + nano::uint128_union ledger; + nano::uint128_union diff; + std::string get_entry () const + { + return boost::str (boost::format ("representative %1% hardcoded %2% ledger %3% mismatch %4%") + % rep.to_account () % hardcoded.format_balance (nano::Mxrb_ratio, 0, true) % ledger.format_balance (nano::Mxrb_ratio, 0, true) % diff.format_balance (nano::Mxrb_ratio, 0, true)); + } + }; + + std::vector mismatched; + mismatched.reserve (hardcoded.size ()); + std::transform (hardcoded.begin (), hardcoded.end (), std::back_inserter (mismatched), [&ledger, &node](auto const & rep) { + auto ledger_rep (ledger.find (rep.first)); + nano::uint128_t ledger_weight = (ledger_rep == ledger.end () ? 0 : ledger_rep->second); + auto absolute = ledger_weight > rep.second ? ledger_weight - rep.second : rep.second - ledger_weight; + return mismatched_t{ rep.first, rep.second, ledger_weight, absolute }; + }); + + // Sort by descending difference + std::sort (mismatched.begin (), mismatched.end (), [](mismatched_t const & left, mismatched_t const & right) { return left.diff > right.diff; }); + + nano::uint128_union const mismatch_total = std::accumulate (mismatched.begin (), mismatched.end (), nano::uint128_t{ 0 }, [](auto sum, mismatched_t const & sample) { return sum + sample.diff.number (); }); + nano::uint128_union const mismatch_mean = mismatch_total.number () / mismatched.size (); + + nano::uint512_union mismatch_variance = std::accumulate (mismatched.begin (), mismatched.end (), nano::uint512_t (0), [M = mismatch_mean.number (), N = mismatched.size ()](nano::uint512_t sum, mismatched_t const & sample) { + auto x = sample.diff.number (); + nano::uint512_t const mean_diff = x > M ? x - M : M - x; + nano::uint512_t const sqr = mean_diff * mean_diff; + return sum + sqr; + }) + / mismatched.size (); + + nano::uint128_union const mismatch_stddev = nano::narrow_cast (boost::multiprecision::sqrt (mismatch_variance.number ())); + + auto const outlier_threshold = std::max (nano::Gxrb_ratio, mismatch_mean.number () + 1 * mismatch_stddev.number ()); + decltype (mismatched) outliers; + std::copy_if (mismatched.begin (), mismatched.end (), std::back_inserter (outliers), [outlier_threshold](mismatched_t const & sample) { + return sample.diff > outlier_threshold; + }); + + auto const newcomer_threshold = std::max (nano::Gxrb_ratio, mismatch_mean.number ()); + std::vector> newcomers; + std::copy_if (ledger.begin (), ledger.end (), std::back_inserter (newcomers), [&hardcoded](auto const & rep) { + return !hardcoded.count (rep.first) && rep.second; + }); + + // Sort by descending weight + std::sort (newcomers.begin (), newcomers.end (), [](auto const & left, auto const & right) { return left.second > right.second; }); + + auto newcomer_entry = [](auto const & rep) { + return boost::str (boost::format ("representative %1% hardcoded --- ledger %2%") % rep.first.to_account () % nano::uint128_union (rep.second).format_balance (nano::Mxrb_ratio, 0, true)); + }; + + std::cout << boost::str (boost::format ("hardcoded weight %1% Mnano\nledger weight %2% Mnano\nmismatched\n\tsamples %3%\n\ttotal %4% Mnano\n\tmean %5% Mnano\n\tsigma %6% Mnano\n") + % total_hardcoded.format_balance (nano::Mxrb_ratio, 0, true) + % total_ledger.format_balance (nano::Mxrb_ratio, 0, true) + % mismatched.size () + % mismatch_total.format_balance (nano::Mxrb_ratio, 0, true) + % mismatch_mean.format_balance (nano::Mxrb_ratio, 0, true) + % mismatch_stddev.format_balance (nano::Mxrb_ratio, 0, true)); + + if (!outliers.empty ()) + { + std::cout << "outliers\n"; + for (auto const & outlier : outliers) + { + std::cout << '\t' << outlier.get_entry () << '\n'; + } + } + + if (!newcomers.empty ()) + { + std::cout << "newcomers\n"; + for (auto const & newcomer : newcomers) + { + if (newcomer.second > newcomer_threshold) + { + std::cout << '\t' << newcomer_entry (newcomer) << '\n'; + } + } + } + + // Log more data + auto const log_threshold = nano::Gxrb_ratio; + for (auto const & sample : mismatched) + { + if (sample.diff > log_threshold) + { + node->logger.always_log (sample.get_entry ()); + } + } + for (auto const & newcomer : newcomers) + { + if (newcomer.second > log_threshold) + { + node->logger.always_log (newcomer_entry (newcomer)); + } + } + } + else + { + std::cout << "Not available for the test network" << std::endl; + result = -1; + } + } else if (vm.count ("debug_block_count")) { auto inactive_node = nano::default_inactive_node (data_path, vm); diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 565d704743..966084e790 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -407,36 +407,19 @@ node_seq (seq) if ((network_params.network.is_live_network () || network_params.network.is_beta_network ()) && !flags.inactive_node) { + auto bootstrap_weights = get_bootstrap_weights (); // Use bootstrap weights if initial bootstrap is not completed - bool use_bootstrap_weight (false); - const uint8_t * weight_buffer = network_params.network.is_live_network () ? nano_bootstrap_weights_live : nano_bootstrap_weights_beta; - size_t weight_size = network_params.network.is_live_network () ? nano_bootstrap_weights_live_size : nano_bootstrap_weights_beta_size; - nano::bufferstream weight_stream ((const uint8_t *)weight_buffer, weight_size); - nano::uint128_union block_height; - if (!nano::try_read (weight_stream, block_height)) + bool use_bootstrap_weight = ledger.cache.block_count < bootstrap_weights.first; + if (use_bootstrap_weight) { - auto max_blocks = (uint64_t)block_height.number (); - use_bootstrap_weight = ledger.cache.block_count < max_blocks; - if (use_bootstrap_weight) + ledger.bootstrap_weight_max_blocks = bootstrap_weights.first; + ledger.bootstrap_weights = bootstrap_weights.second; + for (auto const & rep : ledger.bootstrap_weights) { - ledger.bootstrap_weight_max_blocks = max_blocks; - while (true) - { - nano::account account; - if (nano::try_read (weight_stream, account.bytes)) - { - break; - } - nano::amount weight; - if (nano::try_read (weight_stream, weight.bytes)) - { - break; - } - logger.always_log ("Using bootstrap rep weight: ", account.to_account (), " -> ", weight.format_balance (Mxrb_ratio, 0, true), " XRB"); - ledger.bootstrap_weights[account] = weight.number (); - } + logger.always_log ("Using bootstrap rep weight: ", rep.first.to_account (), " -> ", nano::uint128_union (rep.second).format_balance (Mxrb_ratio, 0, true), " XRB"); } } + // Drop unchecked blocks if initial bootstrap is completed if (!flags.disable_unchecked_drop && !use_bootstrap_weight && !flags.read_only) { @@ -1616,6 +1599,35 @@ void nano::node::epoch_upgrader_impl (nano::private_key const & prv_a, nano::epo logger.always_log ("Epoch upgrade is completed"); } +std::pair nano::node::get_bootstrap_weights () const +{ + std::unordered_map weights; + const uint8_t * weight_buffer = network_params.network.is_live_network () ? nano_bootstrap_weights_live : nano_bootstrap_weights_beta; + size_t weight_size = network_params.network.is_live_network () ? nano_bootstrap_weights_live_size : nano_bootstrap_weights_beta_size; + nano::bufferstream weight_stream ((const uint8_t *)weight_buffer, weight_size); + nano::uint128_union block_height; + uint64_t max_blocks = 0; + if (!nano::try_read (weight_stream, block_height)) + { + max_blocks = nano::narrow_cast (block_height.number ()); + while (true) + { + nano::account account; + if (nano::try_read (weight_stream, account.bytes)) + { + break; + } + nano::amount weight; + if (nano::try_read (weight_stream, weight.bytes)) + { + break; + } + weights[account] = weight.number (); + } + } + return { max_blocks, weights }; +} + nano::inactive_node::inactive_node (boost::filesystem::path const & path_a, nano::node_flags const & node_flags_a) : io_context (std::make_shared ()), alarm (*io_context), diff --git a/nano/node/node.hpp b/nano/node/node.hpp index 838ec6b117..566ab87f66 100644 --- a/nano/node/node.hpp +++ b/nano/node/node.hpp @@ -146,6 +146,7 @@ class node final : public std::enable_shared_from_this bool online () const; bool init_error () const; bool epoch_upgrader (nano::private_key const &, nano::epoch, uint64_t, uint64_t); + std::pair get_bootstrap_weights () const; nano::worker worker; nano::write_database_queue write_database_queue; boost::asio::io_context & io_ctx;