Enabling JournalPruneDataSource

modAionImpl/src/org/aion/zero/impl/AionHub.java

@@ -272,9 +272,39 @@ private void loadBlockchain() {
                 bestBlock != null && // recover only for non-null blocks
                 !this.repository.isValidRoot(bestBlock.getStateRoot())) {
 
+            LOG.info("Recovery initiated due to corrupt world state at block " + bestBlock.getNumber() + ".");
+
             long bestBlockNumber = bestBlock.getNumber();
             byte[] bestBlockRoot = bestBlock.getStateRoot();
 
+            // ensure that the genesis state exists before attempting recovery
+            AionGenesis genesis = cfg.getGenesis();
+            if (!this.repository.isValidRoot(genesis.getStateRoot())) {
+                LOG.info(
+                        "Corrupt world state for genesis block hash: " + genesis.getShortHash() + ", number: " + genesis
+                                .getNumber() + ".");
+                IRepositoryCache track = repository.startTracking();
+
+                Address networkBalanceAddress = PrecompiledContracts.totalCurrencyAddress;
+                track.createAccount(networkBalanceAddress);
+
+                for (Map.Entry<Integer, BigInteger> addr : genesis.getNetworkBalances().entrySet()) {
+                    track.addStorageRow(networkBalanceAddress,
+                                        new DataWord(addr.getKey()),
+                                        new DataWord(addr.getValue()));
+                }
+
+                for (Address addr : genesis.getPremine().keySet()) {
+                    track.createAccount(addr);
+                    track.addBalance(addr, genesis.getPremine().get(addr).getBalance());
+                }
+                track.flush();
+
+                this.repository.commitBlock(genesis.getHeader());
+                this.repository.getBlockStore().saveBlock(genesis, genesis.getCumulativeDifficulty(), true);
+                LOG.info("Rebuilding genesis block SUCCEEDED.");
+            }
+
             recovered = this.blockchain.recoverWorldState(this.repository, bestBlock);
 
             if (!this.repository.isValidRoot(bestBlock.getStateRoot())) {

modAionImpl/src/org/aion/zero/impl/cli/Cli.java

@@ -129,11 +129,52 @@ public int call(final String[] args, final Cfg cfg) {
                         }
                     }
                     break;
+                case "--dump-state-size":
+                    long block_count = 2L;
+
+                    if (args.length < 2) {
+                        System.out.println("Retrieving state size for top " + block_count + " blocks.");
+                        RecoveryUtils.printStateTrieSize(block_count);
+                    } else {
+                        try {
+                            block_count = Long.parseLong(args[1]);
+                        } catch (NumberFormatException e) {
+                            System.out.println("The given argument <" + args[1] + "> cannot be converted to a number.");
+                        }
+                        if (block_count < 1) {
+                            System.out.println("The given argument <" + args[1] + "> is not valid.");
+                            block_count = 2L;
+                        }
+
+                        System.out.println("Retrieving state size for top " + block_count + " blocks.");
+                        RecoveryUtils.printStateTrieSize(block_count);
+                    }
+                    break;
+                case "--dump-state":
+                    long level = -1L;
+
+                    if (args.length < 2) {
+                        System.out.println("Retrieving state for top main chain block...");
+                        RecoveryUtils.printStateTrieDump(level);
+                    } else {
+                        try {
+                            level = Long.parseLong(args[1]);
+                        } catch (NumberFormatException e) {
+                            System.out.println("The given argument <" + args[1] + "> cannot be converted to a number.");
+                        }
+                        if (level == -1L) {
+                            System.out.println("Retrieving state for top main chain block...");
+                        } else {
+                            System.out.println("Retrieving state for main chain block at level " + level + "...");
+                        }
+                        RecoveryUtils.printStateTrieDump(level);
+                    }
+                    break;
                 case "--db-compact":
                     RecoveryUtils.dbCompact();
                     break;
                 case "--dump-blocks":
-                    long count = 100L;
+                    long count = 10L;
 
                     if (args.length < 2) {
                         System.out.println("Printing top " + count + " blocks from database.");
@@ -144,10 +185,14 @@ public int call(final String[] args, final Cfg cfg) {
                         } catch (NumberFormatException e) {
                             System.out.println("The given argument <" + args[1] + "> cannot be converted to a number.");
                         }
+                        if (count < 1) {
+                            System.out.println("The given argument <" + args[1] + "> is not valid.");
+                            count = 10L;
+                        }
+
                         System.out.println("Printing top " + count + " blocks from database.");
                         RecoveryUtils.dumpBlocks(count);
                     }
-                    System.out.println("Finished printing blocks.");
                     break;
                 case "-v":
                     System.out.println("\nVersion");

modAionImpl/src/org/aion/zero/impl/db/AionBlockStore.java

@@ -749,11 +749,14 @@ public BigInteger correctIndexEntry(AionBlock block, BigInteger parentTotalDiffi
         }
     }
 
-    public void dumpPastBlocks(long numberOfBlocks, String reportsFolder) throws IOException {
+    public String dumpPastBlocks(long numberOfBlocks, String reportsFolder) throws IOException {
         lock.readLock().lock();
 
         try {
             long firstBlock = getMaxNumber();
+            if (firstBlock < 0) {
+                return null;
+            }
             long lastBlock = firstBlock - numberOfBlocks;
 
             File file = new File(reportsFolder, System.currentTimeMillis() + "-blocks-report.out");
@@ -786,6 +789,7 @@ public void dumpPastBlocks(long numberOfBlocks, String reportsFolder) throws IOE
             }
 
             writer.close();
+            return file.getName();
         } finally {
             lock.readLock().unlock();
         }

modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java

@@ -76,7 +76,12 @@ private static class AionRepositoryImplHolder {
         // repository singleton instance
         private final static AionRepositoryImpl inst = new AionRepositoryImpl(
                 new RepositoryConfig(new File(config.getBasePath(), config.getDb().getPath()).getAbsolutePath(),
-                                     -1,
+                                     config.getDb().getPrune() > 0 ?
+                                             // if the value is smaller than backward step
+                                             // there is the risk of importing state-less blocks after reboot
+                                             (128 > config.getDb().getPrune() ? 128 : config.getDb().getPrune()) :
+                                             // negative value => pruning disabled
+                                             config.getDb().getPrune(),
                                      ContractDetailsAion.getInstance(),
                                      config.getDb()));
     }
@@ -115,7 +120,7 @@ public TransactionStore<AionTransaction, AionTxReceipt, AionTxInfo> getTransacti
     }
 
     private Trie createStateTrie() {
-        return new SecureTrie(stateDatabase).withPruningEnabled(pruneBlockCount >= 0);
+        return new SecureTrie(stateDSPrune).withPruningEnabled(pruneBlockCount > 0);
     }
 
     @Override
@@ -501,34 +506,31 @@ public void commitBlock(A0BlockHeader blockHeader) {
             worldState.sync();
             detailsDS.syncLargeStorage();
 
-            // temporarily removed since never used
-        /* if (pruneBlockCount >= 0) {
-            stateDSPrune.storeBlockChanges(blockHeader);
-            detailsDS.getStorageDSPrune().storeBlockChanges(blockHeader);
-            pruneBlocks(blockHeader);
-        } */
+            if (pruneBlockCount > 0) {
+                stateDSPrune.storeBlockChanges(blockHeader);
+                detailsDS.getStorageDSPrune().storeBlockChanges(blockHeader);
+                pruneBlocks(blockHeader);
+            }
         } finally {
             rwLock.writeLock().unlock();
         }
     }
 
-    // TODO-AR: reenable state pruning
-    // temporarily removed since never used
-    /* private void pruneBlocks(A0BlockHeader curBlock) {
-        if (curBlock.getNumber() > bestBlockNumber) { // pruning only on
-            // increasing blocks
+    private void pruneBlocks(A0BlockHeader curBlock) {
+        if (curBlock.getNumber() > bestBlockNumber) {
+            // pruning only on increasing blocks
             long pruneBlockNumber = curBlock.getNumber() - pruneBlockCount;
             if (pruneBlockNumber >= 0) {
                 byte[] pruneBlockHash = blockStore.getBlockHashByNumber(pruneBlockNumber);
                 if (pruneBlockHash != null) {
                     A0BlockHeader header = blockStore.getBlockByHash(pruneBlockHash).getHeader();
-                    // stateDSPrune.prune(header);
-                    // detailsDS.getStorageDSPrune().prune(header);
+                    stateDSPrune.prune(header);
+                    detailsDS.getStorageDSPrune().prune(header);
                 }
             }
         }
         bestBlockNumber = curBlock.getNumber();
-    } */
+    }
 
     public Trie getWorldState() {
         return worldState;
@@ -543,7 +545,7 @@ public IRepository getSnapshotTo(byte[] root) {
             repo.blockStore = blockStore;
             repo.cfg = cfg;
             repo.stateDatabase = this.stateDatabase;
-            // repo.stateDSPrune = this.stateDSPrune;
+            repo.stateDSPrune = this.stateDSPrune;
             repo.pruneBlockCount = this.pruneBlockCount;
             repo.detailsDS = this.detailsDS;
             repo.isSnapshot = true;

modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java

@@ -28,8 +28,8 @@
 import org.aion.zero.impl.AionBlockchainImpl;
 import org.aion.zero.impl.config.CfgAion;
 import org.aion.zero.impl.core.IAionBlockchain;
+import org.aion.zero.impl.types.AionBlock;
 
-import java.io.File;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
@@ -138,8 +138,8 @@ public static void dumpBlocks(long count) {
         cfg.getDb().setHeapCacheEnabled(false);
 
         Map<String, String> cfgLog = new HashMap<>();
-        cfgLog.put("DB", "INFO");
-        cfgLog.put("GEN", "INFO");
+        cfgLog.put("DB", "ERROR");
+        cfgLog.put("GEN", "ERROR");
 
         AionLoggerFactory.init(cfgLog);
 
@@ -148,7 +148,12 @@ public static void dumpBlocks(long count) {
 
         AionBlockStore store = repository.getBlockStore();
         try {
-            store.dumpPastBlocks(count, cfg.getBasePath());
+            String file = store.dumpPastBlocks(count, cfg.getBasePath());
+            if (file == null) {
+                System.out.println("The database is empty. Cannot print block information.");
+            } else {
+                System.out.println("Block information stored in " + file);
+            }
         } catch (IOException e) {
             e.printStackTrace();
         }
@@ -203,4 +208,97 @@ public static Status revertTo(IAionBlockchain blockchain, long nbBlock) {
         // ok if we managed to get down to the expected block
         return (nbBestBlock == nbBlock) ? Status.SUCCESS : Status.FAILURE;
     }
+
+    public static void printStateTrieSize(long blockNumber) {
+        // ensure mining is disabled
+        CfgAion cfg = CfgAion.inst();
+        cfg.dbFromXML();
+        cfg.getConsensus().setMining(false);
+
+        Map<String, String> cfgLog = new HashMap<>();
+        cfgLog.put("DB", "ERROR");
+
+        AionLoggerFactory.init(cfgLog);
+
+        // get the current blockchain
+        AionRepositoryImpl repository = AionRepositoryImpl.inst();
+        AionBlockStore store = repository.getBlockStore();
+
+        long topBlock = store.getMaxNumber();
+        if (topBlock < 0) {
+            System.out.println("The database is empty. Cannot print block information.");
+            return;
+        }
+
+        long targetBlock = topBlock - blockNumber + 1;
+        if (targetBlock < 0) {
+            targetBlock = 0;
+        }
+
+        AionBlock block;
+        byte[] stateRoot;
+
+        while (targetBlock <= topBlock) {
+            block = store.getChainBlockByNumber(targetBlock);
+            if (block != null) {
+                stateRoot = block.getStateRoot();
+                try {
+                    System.out.println(
+                            "Block hash: " + block.getShortHash() + ", number: " + block.getNumber() + ", tx count: "
+                                    + block.getTransactionsList().size() + ", state trie kv count = " + repository
+                                    .getWorldState().getTrieSize(stateRoot));
+                } catch (RuntimeException e) {
+                    System.out.println(
+                            "Block hash: " + block.getShortHash() + ", number: " + block.getNumber() + ", tx count: "
+                                    + block.getTransactionsList().size() + ", state trie kv count threw exception: " + e
+                                    .getMessage());
+                }
+            } else {
+                System.out.println("Null block found at level " + targetBlock + ".");
+            }
+            targetBlock++;
+        }
+
+        repository.close();
+    }
+
+    public static void printStateTrieDump(long blockNumber) {
+        // ensure mining is disabled
+        CfgAion cfg = CfgAion.inst();
+        cfg.dbFromXML();
+        cfg.getConsensus().setMining(false);
+
+        Map<String, String> cfgLog = new HashMap<>();
+        cfgLog.put("DB", "ERROR");
+
+        AionLoggerFactory.init(cfgLog);
+
+        // get the current blockchain
+        AionRepositoryImpl repository = AionRepositoryImpl.inst();
+
+        AionBlockStore store = repository.getBlockStore();
+
+        AionBlock block;
+
+        if (blockNumber == -1L) {
+            block = store.getBestBlock();
+            if (block == null) {
+                System.out.println("The requested block does not exist in the database.");
+                return;
+            }
+            blockNumber = block.getNumber();
+        } else {
+            block = store.getChainBlockByNumber(blockNumber);
+            if (block == null) {
+                System.out.println("The requested block does not exist in the database.");
+                return;
+            }
+        }
+
+        byte[] stateRoot = block.getStateRoot();
+        System.out.println("\nBlock hash: " + block.getShortHash() + ", number: " + blockNumber + ", tx count: " + block
+                .getTransactionsList().size() + "\n\n" + repository.getWorldState().getTrieDump(stateRoot));
+
+        repository.close();
+    }
 }