Skip to content

Commit

Permalink
[Pruning] Support pruned chains in RPC & QT wallet (#2977)
Browse files Browse the repository at this point in the history 
* [Pruning] Support pruned chains in RPC & QT wallet
Splitting #2881

* Several RPC calls are modified to support pruned blocks related to main request block.
* RPC “history”: if previous block is pruned, “amount” field is not returned for all blocks, “account” field is not returned for state blocks, also state block type/subtype is “unknown” . If source is pruned, then “account” field is not returned for receive/open legacy & for receive/open state blocks.
* RPC “block_info”/“blocks_info”: if previous block is pruned, “amount” field is not returned. RPC “block_count”: for nodes with enabled pruning returns 2 new fields: “full” blocks & “pruned” blocks. “full” + “pruned” = total “blocks”.
* QT wallet history support for pruned chains

* Remove not yet possible function
* Update tests to cement blocks before pruning
  • Loading branch information
@SergiySW
SergiySW authored Oct 2, 2020
1 parent 22689a7 commit 8462acb
Show file tree
Hide file tree
Showing 4 changed files with 453 additions and 41 deletions.
106 changes: 83 additions & 23 deletions nano/node/json_handler.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1003,8 +1003,12 @@ void nano::json_handler::block_info ()
{
nano::account account (block->account ().is_zero () ? block->sideband ().account : block->account ());
response_l.put ("block_account", account.to_account ());
auto amount (node.ledger.amount (transaction, hash));
response_l.put ("amount", amount.convert_to<std::string> ());
bool error_or_pruned (false);
auto amount (node.ledger.amount_safe (transaction, hash, error_or_pruned));
if (!error_or_pruned)
{
response_l.put ("amount", amount.convert_to<std::string> ());
}
auto balance (node.ledger.balance (transaction, hash));
response_l.put ("balance", balance.convert_to<std::string> ());
response_l.put ("height", std::to_string (block->sideband ().height));
Expand Down Expand Up @@ -1067,11 +1071,15 @@ void nano::json_handler::block_confirm ()
// Trigger callback for confirmed block
node.block_arrival.add (hash);
auto account (node.ledger.account (transaction, hash));
auto amount (node.ledger.amount (transaction, hash));
bool error_or_pruned (false);
auto amount (node.ledger.amount_safe (transaction, hash, error_or_pruned));
bool is_state_send (false);
if (auto state = dynamic_cast<nano::state_block *> (block_l.get ()))
if (!error_or_pruned)
{
is_state_send = node.ledger.is_send (transaction, *state);
if (auto state = dynamic_cast<nano::state_block *> (block_l.get ()))
{
is_state_send = node.ledger.is_send (transaction, *state);
}
}
node.observers.blocks.notify (status, account, amount, is_state_send);
}
Expand Down Expand Up @@ -1153,8 +1161,12 @@ void nano::json_handler::blocks_info ()
boost::property_tree::ptree entry;
nano::account account (block->account ().is_zero () ? block->sideband ().account : block->account ());
entry.put ("block_account", account.to_account ());
auto amount (node.ledger.amount (transaction, hash));
entry.put ("amount", amount.convert_to<std::string> ());
bool error_or_pruned (false);
auto amount (node.ledger.amount_safe (transaction, hash, error_or_pruned));
if (!error_or_pruned)
{
entry.put ("amount", amount.convert_to<std::string> ());
}
auto balance (node.ledger.balance (transaction, hash));
entry.put ("balance", balance.convert_to<std::string> ());
entry.put ("height", std::to_string (block->sideband ().height));
Expand Down Expand Up @@ -1257,6 +1269,11 @@ void nano::json_handler::block_count ()
response_l.put ("count", std::to_string (node.ledger.cache.block_count));
response_l.put ("unchecked", std::to_string (node.store.unchecked_count (node.store.tx_begin_read ())));
response_l.put ("cemented", std::to_string (node.ledger.cache.cemented_count));
if (node.flags.enable_pruning)
{
response_l.put ("full", std::to_string (node.ledger.cache.block_count - node.ledger.cache.pruned_count));
response_l.put ("pruned", std::to_string (node.ledger.cache.pruned_count));
}
response_errors ();
}

Expand Down Expand Up @@ -2180,8 +2197,12 @@ class history_visitor : public nano::block_visitor
tree.put ("type", "send");
auto account (block_a.hashables.destination.to_account ());
tree.put ("account", account);
auto amount (handler.node.ledger.amount (transaction, hash).convert_to<std::string> ());
tree.put ("amount", amount);
bool error_or_pruned (false);
auto amount (handler.node.ledger.amount_safe (transaction, hash, error_or_pruned).convert_to<std::string> ());
if (!error_or_pruned)
{
tree.put ("amount", amount);
}
if (raw)
{
tree.put ("destination", account);
Expand All @@ -2192,10 +2213,17 @@ class history_visitor : public nano::block_visitor
void receive_block (nano::receive_block const & block_a)
{
tree.put ("type", "receive");
auto account (handler.node.ledger.account (transaction, block_a.hashables.source).to_account ());
tree.put ("account", account);
auto amount (handler.node.ledger.amount (transaction, hash).convert_to<std::string> ());
tree.put ("amount", amount);
bool error_or_pruned (false);
auto amount (handler.node.ledger.amount_safe (transaction, hash, error_or_pruned).convert_to<std::string> ());
if (!error_or_pruned)
{
auto source_account (handler.node.ledger.account_safe (transaction, block_a.hashables.source, error_or_pruned));
if (!error_or_pruned)
{
tree.put ("account", source_account.to_account ());
}
tree.put ("amount", amount);
}
if (raw)
{
tree.put ("source", block_a.hashables.source.to_string ());
Expand All @@ -2218,8 +2246,17 @@ class history_visitor : public nano::block_visitor
}
if (block_a.hashables.source != network_params.ledger.genesis_account)
{
tree.put ("account", handler.node.ledger.account (transaction, block_a.hashables.source).to_account ());
tree.put ("amount", handler.node.ledger.amount (transaction, hash).convert_to<std::string> ());
bool error_or_pruned (false);
auto amount (handler.node.ledger.amount_safe (transaction, hash, error_or_pruned).convert_to<std::string> ());
if (!error_or_pruned)
{
auto source_account (handler.node.ledger.account_safe (transaction, block_a.hashables.source, error_or_pruned));
if (!error_or_pruned)
{
tree.put ("account", source_account.to_account ());
}
tree.put ("amount", amount);
}
}
else
{
Expand Down Expand Up @@ -2247,8 +2284,20 @@ class history_visitor : public nano::block_visitor
tree.put ("previous", block_a.hashables.previous.to_string ());
}
auto balance (block_a.hashables.balance.number ());
auto previous_balance (handler.node.ledger.balance (transaction, block_a.hashables.previous));
if (balance < previous_balance)
bool error_or_pruned (false);
auto previous_balance (handler.node.ledger.balance_safe (transaction, block_a.hashables.previous, error_or_pruned));
if (error_or_pruned)
{
if (raw)
{
tree.put ("subtype", "unknown");
}
else
{
tree.put ("type", "unknown");
}
}
else if (balance < previous_balance)
{
if (should_ignore_account (block_a.hashables.link.as_account ()))
{
Expand Down Expand Up @@ -2285,8 +2334,8 @@ class history_visitor : public nano::block_visitor
}
else
{
auto account (handler.node.ledger.account (transaction, block_a.hashables.link.as_block_hash ()));
if (should_ignore_account (account))
auto source_account (handler.node.ledger.account_safe (transaction, block_a.hashables.link.as_block_hash (), error_or_pruned));
if (!error_or_pruned && should_ignore_account (source_account))
{
tree.clear ();
return;
Expand All @@ -2299,7 +2348,10 @@ class history_visitor : public nano::block_visitor
{
tree.put ("type", "receive");
}
tree.put ("account", account.to_account ());
if (!error_or_pruned)
{
tree.put ("account", source_account.to_account ());
}
tree.put ("amount", (balance - previous_balance).convert_to<std::string> ());
}
}
Expand Down Expand Up @@ -3176,14 +3228,15 @@ void nano::json_handler::receive ()
auto block (node.store.block_get (block_transaction, hash));
if (block != nullptr)
{
if (node.store.pending_exists (block_transaction, nano::pending_key (account, hash)))
nano::pending_info pending_info;
if (!node.store.pending_get (block_transaction, nano::pending_key (account, hash), pending_info))
{
auto work (work_optional_impl ());
if (!ec && work)
{
nano::account_info info;
nano::root head;
nano::epoch epoch = block->sideband ().details.epoch;
nano::epoch epoch = pending_info.epoch;
if (!node.store.account_get (block_transaction, account, info))
{
head = info.head;
Expand Down Expand Up @@ -4681,7 +4734,14 @@ void nano::json_handler::wallet_republish ()
{
hashes.push_back (latest);
block = node.store.block_get (block_transaction, latest);
latest = block->previous ();
if (block != nullptr)
{
latest = block->previous ();
}
else
{
latest.clear ();
}
}
std::reverse (hashes.begin (), hashes.end ());
for (auto & hash : hashes)
Expand Down
64 changes: 46 additions & 18 deletions nano/qt/qt.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -522,22 +522,37 @@ class short_text_visitor : public nano::block_visitor
{
type = "Send";
account = block_a.hashables.destination;
amount = ledger.amount (transaction, block_a.hash ());
bool error_or_pruned (false);
amount = ledger.amount_safe (transaction, block_a.hash (), error_or_pruned);
if (error_or_pruned)
{
type = "Send (pruned)";
}
}
void receive_block (nano::receive_block const & block_a)
{
type = "Receive";
account = ledger.account (transaction, block_a.source ());
amount = ledger.amount (transaction, block_a.source ());
bool error_or_pruned (false);
account = ledger.account_safe (transaction, block_a.hashables.source, error_or_pruned);
amount = ledger.amount_safe (transaction, block_a.hash (), error_or_pruned);
if (error_or_pruned)
{
type = "Receive (pruned)";
}
}
void open_block (nano::open_block const & block_a)
{
static nano::network_params params;
type = "Receive";
if (block_a.hashables.source != params.ledger.genesis_account)
{
account = ledger.account (transaction, block_a.hashables.source);
amount = ledger.amount (transaction, block_a.hash ());
bool error_or_pruned (false);
account = ledger.account_safe (transaction, block_a.hashables.source, error_or_pruned);
amount = ledger.amount_safe (transaction, block_a.hash (), error_or_pruned);
if (error_or_pruned)
{
type = "Receive (pruned)";
}
}
else
{
Expand All @@ -554,8 +569,15 @@ class short_text_visitor : public nano::block_visitor
void state_block (nano::state_block const & block_a)
{
auto balance (block_a.hashables.balance.number ());
auto previous_balance (ledger.balance (transaction, block_a.hashables.previous));
if (balance < previous_balance)
bool error_or_pruned (false);
auto previous_balance (ledger.balance_safe (transaction, block_a.hashables.previous, error_or_pruned));
if (error_or_pruned)
{
type = "Unknown (pruned)";
amount = 0;
account = block_a.hashables.account;
}
else if (balance < previous_balance)
{
type = "Send";
amount = previous_balance - balance;
Expand All @@ -576,7 +598,11 @@ class short_text_visitor : public nano::block_visitor
else
{
type = "Receive";
account = ledger.account (transaction, block_a.hashables.link.as_block_hash ());
account = ledger.account_safe (transaction, block_a.hashables.link.as_block_hash (), error_or_pruned);
if (error_or_pruned)
{
type = "Receive (pruned)";
}
}
amount = balance - previous_balance;
}
Expand All @@ -599,16 +625,18 @@ void nano_qt::history::refresh ()
{
QList<QStandardItem *> items;
auto block (ledger.store.block_get (transaction, hash));
debug_assert (block != nullptr);
block->visit (visitor);
items.push_back (new QStandardItem (QString (visitor.type.c_str ())));
items.push_back (new QStandardItem (QString (visitor.account.to_account ().c_str ())));
auto balanceItem = new QStandardItem (QString (wallet.format_balance (visitor.amount).c_str ()));
balanceItem->setData (Qt::AlignRight, Qt::TextAlignmentRole);
items.push_back (balanceItem);
items.push_back (new QStandardItem (QString (hash.to_string ().c_str ())));
hash = block->previous ();
model->appendRow (items);
if (block != nullptr)
{
block->visit (visitor);
items.push_back (new QStandardItem (QString (visitor.type.c_str ())));
items.push_back (new QStandardItem (QString (visitor.account.to_account ().c_str ())));
auto balanceItem = new QStandardItem (QString (wallet.format_balance (visitor.amount).c_str ()));
balanceItem->setData (Qt::AlignRight, Qt::TextAlignmentRole);
items.push_back (balanceItem);
items.push_back (new QStandardItem (QString (hash.to_string ().c_str ())));
hash = block->previous ();
model->appendRow (items);
}
}
}

Expand Down
72 changes: 72 additions & 0 deletions nano/qt_test/qt.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -501,6 +501,78 @@ TEST (history, short_text)
ASSERT_EQ (4, history.model->rowCount ());
}

TEST (history, pruned_source)
{
if (nano::using_rocksdb_in_tests ())
{
// Don't test this in rocksdb mode
return;
}
nano_qt::eventloop_processor processor;
nano::keypair key;
nano::system system (1);
system.wallet (0)->insert_adhoc (key.prv);
nano::account account;
{
auto transaction (system.nodes[0]->wallets.tx_begin_read ());
account = system.account (transaction, 0);
}
auto wallet (std::make_shared<nano_qt::wallet> (*test_application, processor, *system.nodes[0], system.wallet (0), account));
auto store = nano::make_store (system.nodes[0]->logger, nano::unique_path ());
ASSERT_TRUE (!store->init_error ());
nano::genesis genesis;
nano::ledger ledger (*store, system.nodes[0]->stats);
ledger.pruning = true;
nano::block_hash next_pruning;
// Basic pruning for legacy blocks. Previous block is pruned, source is pruned
{
auto transaction (store->tx_begin_write ());
store->initialize (transaction, genesis, ledger.cache);
auto latest (ledger.latest (transaction, nano::dev_genesis_key.pub));
nano::send_block send1 (latest, nano::dev_genesis_key.pub, nano::genesis_amount - 100, nano::dev_genesis_key.prv, nano::dev_genesis_key.pub, *system.work.generate (latest));
ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send1).code);
nano::send_block send2 (send1.hash (), key.pub, nano::genesis_amount - 200, nano::dev_genesis_key.prv, nano::dev_genesis_key.pub, *system.work.generate (send1.hash ()));
ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send2).code);
nano::receive_block receive (send2.hash (), send1.hash (), nano::dev_genesis_key.prv, nano::dev_genesis_key.pub, *system.work.generate (send2.hash ()));
ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, receive).code);
nano::open_block open (send2.hash (), key.pub, key.pub, key.prv, key.pub, *system.work.generate (key.pub));
ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, open).code);
ASSERT_EQ (1, ledger.pruning_action (transaction, send1.hash (), 2));
next_pruning = send2.hash ();
}
nano_qt::history history1 (ledger, nano::dev_genesis_key.pub, *wallet);
history1.refresh ();
ASSERT_EQ (2, history1.model->rowCount ());
nano_qt::history history2 (ledger, key.pub, *wallet);
history2.refresh ();
ASSERT_EQ (1, history2.model->rowCount ());
// Additional legacy test
{
auto transaction (store->tx_begin_write ());
ASSERT_EQ (1, ledger.pruning_action (transaction, next_pruning, 2));
}
history1.refresh ();
ASSERT_EQ (1, history1.model->rowCount ());
history2.refresh ();
ASSERT_EQ (1, history2.model->rowCount ());
// Pruning for state blocks. Previous block is pruned, source is pruned
{
auto transaction (store->tx_begin_write ());
auto latest (ledger.latest (transaction, nano::dev_genesis_key.pub));
nano::state_block send (nano::dev_genesis_key.pub, latest, nano::dev_genesis_key.pub, nano::genesis_amount - 200, key.pub, nano::dev_genesis_key.prv, nano::dev_genesis_key.pub, *system.work.generate (latest));
ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send).code);
auto latest_key (ledger.latest (transaction, key.pub));
nano::state_block receive (key.pub, latest_key, key.pub, 200, send.hash (), key.prv, key.pub, *system.work.generate (latest_key));
ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, receive).code);
ASSERT_EQ (1, ledger.pruning_action (transaction, latest, 2));
ASSERT_EQ (1, ledger.pruning_action (transaction, latest_key, 2));
}
history1.refresh ();
ASSERT_EQ (1, history1.model->rowCount ());
history2.refresh ();
ASSERT_EQ (1, history2.model->rowCount ());
}

TEST (wallet, startup_work)
{
nano_qt::eventloop_processor processor;
Expand Down
Loading

0 comments on commit 8462acb

Please sign in to comment.