From 601e9056c097c9e73609b0cfed807ecfc2a3d7c1 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Fri, 11 May 2018 17:45:31 -0400 Subject: [PATCH 01/35] refactored JournalPruneDataSource to be independent of the block and block header --- .../aion/zero/impl/db/AionRepositoryImpl.java | 189 +++++++++--------- .../org/aion/mcf/db/AbstractRepository.java | 124 ++++++------ .../src/org/aion/mcf/db/DetailsDataStore.java | 75 ++++--- .../aion/mcf/trie/JournalPruneDataSource.java | 28 ++- 4 files changed, 220 insertions(+), 196 deletions(-) diff --git a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java index 057312da49..7b2f58e103 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java +++ b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -19,11 +19,15 @@ * * Contributors: * Aion foundation. - * ******************************************************************************/ - package org.aion.zero.impl.db; +import static org.aion.base.util.ByteUtil.EMPTY_BYTE_ARRAY; +import static org.aion.crypto.HashUtil.EMPTY_TRIE_HASH; + +import java.io.File; +import java.math.BigInteger; +import java.util.*; import org.aion.base.db.*; import org.aion.base.type.Address; import org.aion.base.util.Hex; @@ -42,28 +46,19 @@ import org.aion.zero.types.AionTransaction; import org.aion.zero.types.AionTxReceipt; -import java.io.File; -import java.math.BigInteger; -import java.util.*; - -import static org.aion.base.util.ByteUtil.EMPTY_BYTE_ARRAY; -import static org.aion.crypto.HashUtil.EMPTY_TRIE_HASH; - -/** - * Has direct database connection. - */ -public class AionRepositoryImpl extends AbstractRepository { +/** Has direct database connection. */ +public class AionRepositoryImpl + extends AbstractRepository { private TransactionStore transactionStore; /** * used by getSnapShotTo * - * @ATTENTION: when do snap shot, another instance will be created. Make - * sure it is used only by getSnapShotTo + *

@ATTENTION: when do snap shot, another instance will be created. Make sure it is used only + * by getSnapShotTo */ - protected AionRepositoryImpl() { - } + protected AionRepositoryImpl() {} protected AionRepositoryImpl(IRepositoryConfig repoConfig) { this.cfg = repoConfig; @@ -74,16 +69,24 @@ private static class AionRepositoryImplHolder { // configuration private static CfgAion config = CfgAion.inst(); // repository singleton instance - private final static AionRepositoryImpl inst = new AionRepositoryImpl( - new RepositoryConfig(new File(config.getBasePath(), config.getDb().getPath()).getAbsolutePath(), - config.getDb().getPrune() > 0 ? - // if the value is smaller than backward step - // there is the risk of importing state-less blocks after reboot - (128 > config.getDb().getPrune() ? 128 : config.getDb().getPrune()) : - // negative value => pruning disabled - config.getDb().getPrune(), - ContractDetailsAion.getInstance(), - config.getDb())); + private static final AionRepositoryImpl inst = + new AionRepositoryImpl( + new RepositoryConfig( + new File(config.getBasePath(), config.getDb().getPath()) + .getAbsolutePath(), + config.getDb().getPrune() > 0 + ? + // if the value is smaller than backward step + // there is the risk of importing state-less blocks after + // reboot + (128 > config.getDb().getPrune() + ? 128 + : config.getDb().getPrune()) + : + // negative value => pruning disabled + config.getDb().getPrune(), + ContractDetailsAion.getInstance(), + config.getDb())); } public static AionRepositoryImpl inst() { @@ -99,8 +102,9 @@ private void init() { initializeDatabasesAndCaches(); // Setup the cache for transaction data source. - this.transactionStore = new TransactionStore<>(transactionDatabase, - AionTransactionStoreSerializer.serializer); + this.transactionStore = + new TransactionStore<>( + transactionDatabase, AionTransactionStoreSerializer.serializer); // Setup block store. this.blockStore = new AionBlockStore(indexDatabase, blockDatabase, checkIntegrity); @@ -112,9 +116,7 @@ private void init() { } } - /** - * @implNote The transaction store is not locked within the repository implementation. - */ + /** @implNote The transaction store is not locked within the repository implementation. */ public TransactionStore getTransactionStore() { return this.transactionStore; } @@ -124,7 +126,8 @@ private Trie createStateTrie() { } @Override - public void updateBatch(Map stateCache, + public void updateBatch( + Map stateCache, Map> detailsCache) { rwLock.writeLock().lock(); @@ -153,17 +156,19 @@ public void updateBatch(Map stateCache, updateAccountState(address, accountState); if (LOG.isTraceEnabled()) { - LOG.trace("update: [{}],nonce: [{}] balance: [{}] [{}]", - Hex.toHexString(address.toBytes()), - accountState.getNonce(), - accountState.getBalance(), - Hex.toHexString(contractDetails.getStorageHash())); + LOG.trace( + "update: [{}],nonce: [{}] balance: [{}] [{}]", + Hex.toHexString(address.toBytes()), + accountState.getNonce(), + accountState.getBalance(), + Hex.toHexString(contractDetails.getStorageHash())); } } continue; } - ContractDetailsCacheImpl contractDetailsCache = (ContractDetailsCacheImpl) contractDetails; + ContractDetailsCacheImpl contractDetailsCache = + (ContractDetailsCacheImpl) contractDetails; if (contractDetailsCache.origContract == null) { contractDetailsCache.origContract = this.cfg.contractDetailsImpl(); @@ -171,7 +176,8 @@ public void updateBatch(Map stateCache, contractDetailsCache.origContract.setAddress(address); } catch (Exception e) { e.printStackTrace(); - LOG.error("contractDetailsCache setAddress exception [{}]", e.toString()); + LOG.error( + "contractDetailsCache setAddress exception [{}]", e.toString()); } contractDetailsCache.commit(); @@ -188,11 +194,12 @@ public void updateBatch(Map stateCache, updateAccountState(address, accountState); if (LOG.isTraceEnabled()) { - LOG.trace("update: [{}],nonce: [{}] balance: [{}] [{}]", - Hex.toHexString(address.toBytes()), - accountState.getNonce(), - accountState.getBalance(), - Hex.toHexString(contractDetails.getStorageHash())); + LOG.trace( + "update: [{}],nonce: [{}] balance: [{}] [{}]", + Hex.toHexString(address.toBytes()), + accountState.getNonce(), + accountState.getBalance(), + Hex.toHexString(contractDetails.getStorageHash())); } } } @@ -205,10 +212,9 @@ public void updateBatch(Map stateCache, } } - /** - * @implNote The method calling this method must handle the locking. - */ - private void updateContractDetails(final Address address, final IContractDetails contractDetails) { + /** @implNote The method calling this method must handle the locking. */ + private void updateContractDetails( + final Address address, final IContractDetails contractDetails) { // locked by calling method detailsDS.update(address, contractDetails); } @@ -369,9 +375,7 @@ public BigInteger getNonce(Address address) { return (account == null) ? BigInteger.ZERO : account.getNonce(); } - /** - * @implNote The method calling this method must handle the locking. - */ + /** @implNote The method calling this method must handle the locking. */ private void updateAccountState(Address address, AccountState accountState) { // locked by calling method worldState.update(address.toBytes(), accountState.getEncoded()); @@ -379,10 +383,10 @@ private void updateAccountState(Address address, AccountState accountState) { /** * @inheritDoc - * @implNote Any other method calling this can rely on the fact that - * the contract details returned is a newly created object by {@link IContractDetails#getSnapshotTo(byte[])}. - * Since this querying method it locked, the methods calling it - * may not need to be locked or synchronized, depending on the specific use case. + * @implNote Any other method calling this can rely on the fact that the contract details + * returned is a newly created object by {@link IContractDetails#getSnapshotTo(byte[])}. + * Since this querying method it locked, the methods calling it may not need to be locked + * or synchronized, depending on the specific use case. */ @Override public IContractDetails getContractDetails(Address address) { @@ -423,10 +427,9 @@ public boolean hasContractDetails(Address address) { /** * @inheritDoc - * @implNote Any other method calling this can rely on the fact that - * the account state returned is a newly created object. - * Since this querying method it locked, the methods calling it - * may not need to be locked or synchronized, depending on the specific use case. + * @implNote Any other method calling this can rely on the fact that the account state returned + * is a newly created object. Since this querying method it locked, the methods calling it + * may not need to be locked or synchronized, depending on the specific use case. */ @Override public AccountState getAccountState(Address address) { @@ -439,7 +442,8 @@ public AccountState getAccountState(Address address) { if (accountData.length != 0) { result = new AccountState(accountData); - LOG.debug("New AccountSate [{}], State [{}]", address.toString(), result.toString()); + LOG.debug( + "New AccountSate [{}], State [{}]", address.toString(), result.toString()); } return result; } finally { @@ -453,11 +457,13 @@ public boolean hasAccountState(Address address) { } /** - * @implNote The loaded objects are fresh copies of the original account - * state and contract details. + * @implNote The loaded objects are fresh copies of the original account state and contract + * details. */ @Override - public void loadAccountState(Address address, Map cacheAccounts, + public void loadAccountState( + Address address, + Map cacheAccounts, Map> cacheDetails) { AccountState account = getAccountState(address); @@ -507,8 +513,10 @@ public void commitBlock(A0BlockHeader blockHeader) { detailsDS.syncLargeStorage(); if (pruneBlockCount > 0) { - stateDSPrune.storeBlockChanges(blockHeader); - detailsDS.getStorageDSPrune().storeBlockChanges(blockHeader); + stateDSPrune.storeBlockChanges(blockHeader.getHash(), blockHeader.getNumber()); + detailsDS + .getStorageDSPrune() + .storeBlockChanges(blockHeader.getHash(), blockHeader.getNumber()); pruneBlocks(blockHeader); } } finally { @@ -524,8 +532,8 @@ private void pruneBlocks(A0BlockHeader curBlock) { byte[] pruneBlockHash = blockStore.getBlockHashByNumber(pruneBlockNumber); if (pruneBlockHash != null) { A0BlockHeader header = blockStore.getBlockByHash(pruneBlockHash).getHeader(); - stateDSPrune.prune(header); - detailsDS.getStorageDSPrune().prune(header); + stateDSPrune.prune(header.getHash(), header.getNumber()); + detailsDS.getStorageDSPrune().prune(header.getHash(), header.getNumber()); } } } @@ -597,10 +605,7 @@ public void removeTxBatch(Set clearTxSet, boolean isPool) { } } - /** - * This function cannot for any reason fail, otherwise we may have dangling - * file IO locks - */ + /** This function cannot for any reason fail, otherwise we may have dangling file IO locks */ @Override public void close() { rwLock.writeLock().lock(); @@ -662,7 +667,8 @@ public void close() { pendingTxCacheDatabase = null; } } catch (Exception e) { - LOGGEN.error("Exception occurred while closing the pendingTxCacheDatabase store.", e); + LOGGEN.error( + "Exception occurred while closing the pendingTxCacheDatabase store.", e); } } finally { rwLock.writeLock().unlock(); @@ -670,12 +676,11 @@ public void close() { } /** - * Retrieves the underlying state database that sits below all caches. This - * is usually provided by {@link org.aion.db.impl.leveldb.LevelDB} or - * {@link org.aion.db.impl.leveldb.LevelDB}. - *

- * Note that referencing the state database directly is unsafe, and should - * only be used for debugging and testing purposes. + * Retrieves the underlying state database that sits below all caches. This is usually provided + * by {@link org.aion.db.impl.leveldb.LevelDB} or {@link org.aion.db.impl.leveldb.LevelDB}. + * + *

Note that referencing the state database directly is unsafe, and should only be used for + * debugging and testing purposes. * * @return */ @@ -684,12 +689,11 @@ public IByteArrayKeyValueDatabase getStateDatabase() { } /** - * Retrieves the underlying details database that sits below all caches. - * This is usually provided by {@link org.aion.db.impl.mockdb.MockDB} - * or {@link org.aion.db.impl.mockdb.MockDB}. - *

- * Note that referencing the state database directly is unsafe, and should - * only be used for debugging and testing purposes. + * Retrieves the underlying details database that sits below all caches. This is usually + * provided by {@link org.aion.db.impl.mockdb.MockDB} or {@link org.aion.db.impl.mockdb.MockDB}. + * + *

Note that referencing the state database directly is unsafe, and should only be used for + * debugging and testing purposes. * * @return */ @@ -697,17 +701,20 @@ public IByteArrayKeyValueDatabase getDetailsDatabase() { return this.detailsDatabase; } - /** - * For testing. - */ + /** For testing. */ public IByteArrayKeyValueDatabase getIndexDatabase() { return this.indexDatabase; } @Override public String toString() { - return "AionRepositoryImpl{ identityHashCode=" + System.identityHashCode(this) + ", " + // - "databaseGroupSize=" + (databaseGroup == null ? 0 : databaseGroup.size()) + '}'; + return "AionRepositoryImpl{ identityHashCode=" + + System.identityHashCode(this) + + ", " + + // + "databaseGroupSize=" + + (databaseGroup == null ? 0 : databaseGroup.size()) + + '}'; } @Override diff --git a/modMcf/src/org/aion/mcf/db/AbstractRepository.java b/modMcf/src/org/aion/mcf/db/AbstractRepository.java index f894acd753..269727801b 100644 --- a/modMcf/src/org/aion/mcf/db/AbstractRepository.java +++ b/modMcf/src/org/aion/mcf/db/AbstractRepository.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * * Copyright (c) 2017, 2018 Aion foundation. * @@ -18,9 +18,17 @@ * Contributors: * Aion foundation. *******************************************************************************/ - package org.aion.mcf.db; +import static org.aion.db.impl.DatabaseFactory.Props; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Objects; +import java.util.Properties; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.base.db.IRepository; import org.aion.base.db.IRepositoryConfig; @@ -38,22 +46,13 @@ import org.aion.mcf.vm.types.DataWord; import org.slf4j.Logger; -import java.io.File; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Objects; -import java.util.Properties; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import static org.aion.db.impl.DatabaseFactory.Props; - -//import org.aion.dbmgr.exception.DriverManagerNoSuitableDriverRegisteredException; +// import org.aion.dbmgr.exception.DriverManagerNoSuitableDriverRegisteredException; -/** - * Abstract Repository class. - */ -public abstract class AbstractRepository, BH extends IBlockHeader, BSB extends IBlockStoreBase> +/** Abstract Repository class. */ +public abstract class AbstractRepository< + BLK extends AbstractBlock, + BH extends IBlockHeader, + BSB extends IBlockStoreBase> implements IRepository { // Logger @@ -63,9 +62,9 @@ public abstract class AbstractRepository databaseGroup; - protected JournalPruneDataSource stateDSPrune; + protected JournalPruneDataSource stateDSPrune; protected DetailsDataStore detailsDS; // Read Write Lock @@ -125,18 +124,18 @@ protected void initializeDatabasesAndCaches() throws Exception { * on startup, enforce conditions here for safety */ Objects.requireNonNull(this.cfg); -// Objects.requireNonNull(this.cfg.getVendorList()); -// Objects.requireNonNull(this.cfg.getActiveVendor()); - -// /** -// * TODO: this is hack There should be some information on the -// * persistence of the DB so that we do not have to manually check. -// * Currently this information exists within -// * {@link DBVendor#getPersistence()}, but is not utilized. -// */ -// if (this.cfg.getActiveVendor().equals(DBVendor.MOCKDB.toValue())) { -// LOG.warn("WARNING: Active vendor is set to MockDB, data will not persist"); -// } else { + // Objects.requireNonNull(this.cfg.getVendorList()); + // Objects.requireNonNull(this.cfg.getActiveVendor()); + + // /** + // * TODO: this is hack There should be some information on the + // * persistence of the DB so that we do not have to manually check. + // * Currently this information exists within + // * {@link DBVendor#getPersistence()}, but is not utilized. + // */ + // if (this.cfg.getActiveVendor().equals(DBVendor.MOCKDB.toValue())) { + // LOG.warn("WARNING: Active vendor is set to MockDB, data will not persist"); + // } else { // verify user-provided path File f = new File(this.cfg.getDbPath()); try { @@ -148,22 +147,27 @@ protected void initializeDatabasesAndCaches() throws Exception { f.mkdirs(); } } catch (Exception e) { - throw new InvalidFilePathException("Resolved file path \"" + this.cfg.getDbPath() - + "\" not valid as reported by the OS or a read/write permissions error occurred. Please provide an alternative DB file path in /config/config.xml."); + throw new InvalidFilePathException( + "Resolved file path \"" + + this.cfg.getDbPath() + + "\" not valid as reported by the OS or a read/write permissions error occurred. Please provide an alternative DB file path in /config/config.xml."); } -// } -// -// if (!Arrays.asList(this.cfg.getVendorList()).contains(this.cfg.getActiveVendor())) { -// -// ArrayList vendorListString = new ArrayList<>(); -// for (String v : this.cfg.getVendorList()) { -// vendorListString.add("\"" + v + "\""); -// } -// throw new DriverManagerNoSuitableDriverRegisteredException( -// "Please check the vendor name field in /config/config.xml.\n" -// + "No suitable driver found with name \"" + this.cfg.getActiveVendor() -// + "\".\nPlease select a driver from the following vendor list: " + vendorListString); -// } + // } + // + // if (!Arrays.asList(this.cfg.getVendorList()).contains(this.cfg.getActiveVendor())) + // { + // + // ArrayList vendorListString = new ArrayList<>(); + // for (String v : this.cfg.getVendorList()) { + // vendorListString.add("\"" + v + "\""); + // } + // throw new DriverManagerNoSuitableDriverRegisteredException( + // "Please check the vendor name field in /config/config.xml.\n" + // + "No suitable driver found with name \"" + + // this.cfg.getActiveVendor() + // + "\".\nPlease select a driver from the following vendor list: + // " + vendorListString); + // } Properties sharedProps; @@ -171,8 +175,10 @@ protected void initializeDatabasesAndCaches() throws Exception { try { databaseGroup = new ArrayList<>(); - checkIntegrity = Boolean - .valueOf(cfg.getDatabaseConfig(CfgDb.Names.DEFAULT).getProperty(Props.CHECK_INTEGRITY)); + checkIntegrity = + Boolean.valueOf( + cfg.getDatabaseConfig(CfgDb.Names.DEFAULT) + .getProperty(Props.CHECK_INTEGRITY)); // getting state specific properties sharedProps = cfg.getDatabaseConfig(STATE_DB); @@ -241,7 +247,7 @@ protected void initializeDatabasesAndCaches() throws Exception { // Setup the cache for transaction data source. this.detailsDS = new DetailsDataStore<>(detailsDatabase, storageDatabase, this.cfg); - stateDSPrune = new JournalPruneDataSource<>(stateDatabase); + stateDSPrune = new JournalPruneDataSource(stateDatabase); pruneBlockCount = pruneEnabled ? this.cfg.getPrune() : -1; if (pruneEnabled && pruneBlockCount > 0) { LOGGEN.info("Pruning block count set to {}.", pruneBlockCount); @@ -272,16 +278,18 @@ private IByteArrayKeyValueDatabase connectAndOpen(Properties info) { // check object status if (db == null) { - LOG.error("Database <{}> connection could not be established for <{}>.", - info.getProperty(Props.DB_TYPE), - info.getProperty(Props.DB_NAME)); + LOG.error( + "Database <{}> connection could not be established for <{}>.", + info.getProperty(Props.DB_TYPE), + info.getProperty(Props.DB_NAME)); } // check persistence status if (!db.isCreatedOnDisk()) { - LOG.error("Database <{}> cannot be saved to disk for <{}>.", - info.getProperty(Props.DB_TYPE), - info.getProperty(Props.DB_NAME)); + LOG.error( + "Database <{}> cannot be saved to disk for <{}>.", + info.getProperty(Props.DB_TYPE), + info.getProperty(Props.DB_NAME)); } return db; diff --git a/modMcf/src/org/aion/mcf/db/DetailsDataStore.java b/modMcf/src/org/aion/mcf/db/DetailsDataStore.java index c97997f95f..280cd38787 100644 --- a/modMcf/src/org/aion/mcf/db/DetailsDataStore.java +++ b/modMcf/src/org/aion/mcf/db/DetailsDataStore.java @@ -1,25 +1,42 @@ -/******************************************************************************* +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. * - * Copyright (c) 2017, 2018 Aion foundation. + * This file is part of the aion network project. * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see + * along with the aion network project source files. + * If not, see . * - * Contributors: + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: * Aion foundation. - *******************************************************************************/ + * team through the ethereumJ library. + * Ether.Camp Inc. (US) team through Ethereum Harmony. + * John Tromp through the Equihash solver. + * Samuel Neves through the BLAKE2 implementation. + * Zcash project team. + * Bitcoinj team. + ******************************************************************************/ package org.aion.mcf.db; +import static org.aion.base.util.ByteArrayWrapper.wrap; + +import java.util.*; import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.base.db.IContractDetails; import org.aion.base.db.IRepositoryConfig; @@ -31,43 +48,38 @@ import org.aion.mcf.types.AbstractBlock; import org.aion.mcf.vm.types.DataWord; -import java.util.*; - -import static org.aion.base.util.ByteArrayWrapper.wrap; - -/** - * Detail data storage , - */ -public class DetailsDataStore, BH extends IBlockHeader> { +/** Detail data storage , */ +public class DetailsDataStore< + BLK extends AbstractBlock, BH extends IBlockHeader> { - private JournalPruneDataSource storageDSPrune; + private JournalPruneDataSource storageDSPrune; private IRepositoryConfig repoConfig; private IByteArrayKeyValueDatabase detailsSrc; private IByteArrayKeyValueDatabase storageSrc; private Set removes = new HashSet<>(); - public DetailsDataStore() { - } + public DetailsDataStore() {} - public DetailsDataStore(IByteArrayKeyValueDatabase detailsCache, IByteArrayKeyValueDatabase storageCache, + public DetailsDataStore( + IByteArrayKeyValueDatabase detailsCache, + IByteArrayKeyValueDatabase storageCache, IRepositoryConfig repoConfig) { this.repoConfig = repoConfig; withDb(detailsCache, storageCache); } - public DetailsDataStore withDb(IByteArrayKeyValueDatabase detailsSrc, - IByteArrayKeyValueDatabase storageSrc) { + public DetailsDataStore withDb( + IByteArrayKeyValueDatabase detailsSrc, IByteArrayKeyValueDatabase storageSrc) { this.detailsSrc = detailsSrc; this.storageSrc = storageSrc; - this.storageDSPrune = new JournalPruneDataSource<>(storageSrc); + this.storageDSPrune = new JournalPruneDataSource(storageSrc); return this; } /** - * Fetches the ContractDetails from the cache, and if it doesn't exist, add - * to the remove set. + * Fetches the ContractDetails from the cache, and if it doesn't exist, add to the remove set. * * @param key * @return @@ -110,7 +122,6 @@ public synchronized void update(Address key, IContractDetails contract // Remove from the remove set. removes.remove(wrappedKey); - } public synchronized void remove(byte[] key) { @@ -170,7 +181,7 @@ public void syncLargeStorage() { } } - public JournalPruneDataSource getStorageDSPrune() { + public JournalPruneDataSource getStorageDSPrune() { return storageDSPrune; } diff --git a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java index f547901dfd..f5dc2f116a 100644 --- a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java +++ b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java @@ -37,8 +37,6 @@ import java.util.*; import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.base.db.IByteArrayKeyValueStore; -import org.aion.base.type.IBlock; -import org.aion.base.type.IBlockHeader; import org.aion.base.util.ByteArrayWrapper; /** @@ -48,12 +46,11 @@ * submitted to the underlying DataSource with respect to following inserts. E.g. if the key was * deleted at block N and then inserted at block N + 10 this delete is not passed. */ -public class JournalPruneDataSource, BH extends IBlockHeader> - implements IByteArrayKeyValueStore { +public class JournalPruneDataSource implements IByteArrayKeyValueStore { private class Updates { - - BH blockHeader; + ByteArrayWrapper blockHeader; + long blockNumber; Set insertedKeys = new HashSet<>(); Set deletedKeys = new HashSet<>(); } @@ -157,20 +154,22 @@ private Ref decRef(ByteArrayWrapper keyW) { return cnt; } - public synchronized void storeBlockChanges(BH header) { + public synchronized void storeBlockChanges(byte[] blockHash, long blockNumber) { if (!enabled) { return; } - currentUpdates.blockHeader = header; - blockUpdates.put(new ByteArrayWrapper(header.getHash()), currentUpdates); + ByteArrayWrapper hash = new ByteArrayWrapper(blockHash); + currentUpdates.blockHeader = hash; + currentUpdates.blockNumber = blockNumber; + blockUpdates.put(hash, currentUpdates); currentUpdates = new Updates(); } - public synchronized void prune(BH header) { + public synchronized void prune(byte[] blockHash, long blockNumber) { if (!enabled) { return; } - ByteArrayWrapper blockHashW = new ByteArrayWrapper(header.getHash()); + ByteArrayWrapper blockHashW = new ByteArrayWrapper(blockHash); Updates updates = blockUpdates.remove(blockHashW); if (updates != null) { for (ByteArrayWrapper insertedKey : updates.insertedKeys) { @@ -188,20 +187,19 @@ public synchronized void prune(BH header) { } src.deleteBatch(batchRemove); - rollbackForkBlocks(header.getNumber()); + rollbackForkBlocks(blockNumber); } } private void rollbackForkBlocks(long blockNum) { for (Updates updates : new ArrayList<>(blockUpdates.values())) { - if (updates.blockHeader.getNumber() == blockNum) { + if (updates.blockNumber == blockNum) { rollback(updates.blockHeader); } } } - private synchronized void rollback(BH header) { - ByteArrayWrapper blockHashW = new ByteArrayWrapper(header.getHash()); + private synchronized void rollback(ByteArrayWrapper blockHashW) { Updates updates = blockUpdates.remove(blockHashW); Map batchRemove = new HashMap<>(); for (ByteArrayWrapper insertedKey : updates.insertedKeys) { From e92d842128b3fabd5d7e159812ad4fb7d05b6fab Mon Sep 17 00:00:00 2001 From: Alexandra Date: Mon, 14 May 2018 11:44:00 -0400 Subject: [PATCH 02/35] replacing synchronized methods with read-write locks --- .../aion/mcf/trie/JournalPruneDataSource.java | 264 ++++++++++++------ 1 file changed, 184 insertions(+), 80 deletions(-) diff --git a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java index f5dc2f116a..a2257a3d5c 100644 --- a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java +++ b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java @@ -35,9 +35,14 @@ package org.aion.mcf.trie; import java.util.*; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.base.db.IByteArrayKeyValueStore; import org.aion.base.util.ByteArrayWrapper; +import org.aion.log.AionLoggerFactory; +import org.aion.log.LogEnum; +import org.slf4j.Logger; /** * The DataSource which doesn't immediately forward delete updates (unlike inserts) but collects @@ -48,6 +53,9 @@ */ public class JournalPruneDataSource implements IByteArrayKeyValueStore { + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private static final Logger LOG = AionLoggerFactory.getLogger(LogEnum.DB.name()); + private class Updates { ByteArrayWrapper blockHeader; long blockNumber; @@ -82,58 +90,99 @@ public JournalPruneDataSource(IByteArrayKeyValueDatabase src) { } public void setPruneEnabled(boolean e) { + lock.writeLock().lock(); + enabled = e; + + lock.writeLock().unlock(); } - public synchronized void put(byte[] key, byte[] value) { - ByteArrayWrapper keyW = new ByteArrayWrapper(key); + public void put(byte[] key, byte[] value) { + lock.writeLock().lock(); - // Check to see the value exists. - if (value != null) { + try { + ByteArrayWrapper keyW = new ByteArrayWrapper(key); - // If it exists and pruning is enabled. - if (enabled) { - currentUpdates.insertedKeys.add(keyW); - incRef(keyW); - } + // Check to see the value exists. + if (value != null) { - // put to source database. - src.put(key, value); + // If it exists and pruning is enabled. + if (enabled) { + currentUpdates.insertedKeys.add(keyW); + incRef(keyW); + } + + // put to source database. + src.put(key, value); - } else { - // Value does not exist, so we delete from current updates - if (enabled) { - currentUpdates.deletedKeys.add(keyW); + } else { + // Value does not exist, so we delete from current updates + if (enabled) { + currentUpdates.deletedKeys.add(keyW); + } + // delete is not sent to source db } - // delete is not sent to source db + } catch (Exception e) { + if (e instanceof RuntimeException) { + throw e; + } else { + LOG.error("Could not put key-value pair due to ", e); + } + } finally { + lock.writeLock().unlock(); } } - public synchronized void delete(byte[] key) { - if (!enabled) { - return; + public void delete(byte[] key) { + lock.writeLock().lock(); + + try { + if (!enabled) { + return; + } + currentUpdates.deletedKeys.add(new ByteArrayWrapper(key)); + // delete is delayed + } catch (Exception e) { + if (e instanceof RuntimeException) { + throw e; + } else { + LOG.error("Could not delete key due to ", e); + } + } finally { + lock.writeLock().unlock(); } - currentUpdates.deletedKeys.add(new ByteArrayWrapper(key)); - // delete is delayed } - public synchronized void updateBatch(Map rows) { - Map insertsOnly = new HashMap<>(); - for (Map.Entry entry : rows.entrySet()) { - ByteArrayWrapper keyW = new ByteArrayWrapper(entry.getKey()); - if (entry.getValue() != null) { - if (enabled) { - currentUpdates.insertedKeys.add(keyW); - incRef(keyW); + @Override + public void putBatch(Map inputMap) { + lock.writeLock().lock(); + + try { + Map insertsOnly = new HashMap<>(); + for (Map.Entry entry : inputMap.entrySet()) { + ByteArrayWrapper keyW = new ByteArrayWrapper(entry.getKey()); + if (entry.getValue() != null) { + if (enabled) { + currentUpdates.insertedKeys.add(keyW); + incRef(keyW); + } + insertsOnly.put(entry.getKey(), entry.getValue()); + } else { + if (enabled) { + currentUpdates.deletedKeys.add(keyW); + } } - insertsOnly.put(entry.getKey(), entry.getValue()); + } + src.putBatch(insertsOnly); + } catch (Exception e) { + if (e instanceof RuntimeException) { + throw e; } else { - if (enabled) { - currentUpdates.deletedKeys.add(keyW); - } + LOG.error("Could not put batch due to ", e); } + } finally { + lock.writeLock().unlock(); } - src.putBatch(insertsOnly); } private void incRef(ByteArrayWrapper keyW) { @@ -154,40 +203,52 @@ private Ref decRef(ByteArrayWrapper keyW) { return cnt; } - public synchronized void storeBlockChanges(byte[] blockHash, long blockNumber) { - if (!enabled) { - return; + public void storeBlockChanges(byte[] blockHash, long blockNumber) { + lock.writeLock().lock(); + + try { + if (!enabled) { + return; + } + ByteArrayWrapper hash = new ByteArrayWrapper(blockHash); + currentUpdates.blockHeader = hash; + currentUpdates.blockNumber = blockNumber; + blockUpdates.put(hash, currentUpdates); + currentUpdates = new Updates(); + } finally { + lock.writeLock().unlock(); } - ByteArrayWrapper hash = new ByteArrayWrapper(blockHash); - currentUpdates.blockHeader = hash; - currentUpdates.blockNumber = blockNumber; - blockUpdates.put(hash, currentUpdates); - currentUpdates = new Updates(); } - public synchronized void prune(byte[] blockHash, long blockNumber) { - if (!enabled) { - return; - } - ByteArrayWrapper blockHashW = new ByteArrayWrapper(blockHash); - Updates updates = blockUpdates.remove(blockHashW); - if (updates != null) { - for (ByteArrayWrapper insertedKey : updates.insertedKeys) { - decRef(insertedKey).dbRef = true; + public void prune(byte[] blockHash, long blockNumber) { + lock.writeLock().lock(); + + try { + if (!enabled) { + return; } + ByteArrayWrapper blockHashW = new ByteArrayWrapper(blockHash); + Updates updates = blockUpdates.remove(blockHashW); + if (updates != null) { + for (ByteArrayWrapper insertedKey : updates.insertedKeys) { + decRef(insertedKey).dbRef = true; + } - List batchRemove = new ArrayList<>(); - for (ByteArrayWrapper key : updates.deletedKeys) { - Ref ref = refCount.get(key); - if (ref == null || ref.journalRefs == 0) { - batchRemove.add(key.getData()); - } else if (ref != null) { - ref.dbRef = false; + List batchRemove = new ArrayList<>(); + for (ByteArrayWrapper key : updates.deletedKeys) { + Ref ref = refCount.get(key); + if (ref == null || ref.journalRefs == 0) { + batchRemove.add(key.getData()); + } else if (ref != null) { + ref.dbRef = false; + } } - } - src.deleteBatch(batchRemove); + src.deleteBatch(batchRemove); - rollbackForkBlocks(blockNumber); + rollbackForkBlocks(blockNumber); + } + } finally { + lock.writeLock().unlock(); } } @@ -199,7 +260,7 @@ private void rollbackForkBlocks(long blockNum) { } } - private synchronized void rollback(ByteArrayWrapper blockHashW) { + private void rollback(ByteArrayWrapper blockHashW) { Updates updates = blockUpdates.remove(blockHashW); Map batchRemove = new HashMap<>(); for (ByteArrayWrapper insertedKey : updates.insertedKeys) { @@ -220,29 +281,54 @@ public LinkedHashMap getBlockUpdates() { } public int getDeletedKeysCount() { - return currentUpdates.deletedKeys.size(); + lock.readLock().lock(); + try { + return currentUpdates.deletedKeys.size(); + } finally { + lock.readLock().unlock(); + } } public int getInsertedKeysCount() { - return currentUpdates.insertedKeys.size(); + lock.readLock().lock(); + try { + return currentUpdates.insertedKeys.size(); + } finally { + lock.readLock().unlock(); + } } public Optional get(byte[] key) { - return src.get(key); + lock.readLock().lock(); + try { + return src.get(key); + } catch (Exception e) { + throw e; + } finally { + lock.readLock().unlock(); + } } public Set keys() { - return src.keys(); + lock.readLock().lock(); + try { + return src.keys(); + } catch (Exception e) { + throw e; + } finally { + lock.readLock().unlock(); + } } @Override public void close() { - src.close(); - } + lock.writeLock().lock(); - @Override - public void putBatch(Map inputMap) { - updateBatch(inputMap); + try { + src.close(); + } finally { + lock.writeLock().unlock(); + } } @Override @@ -257,20 +343,38 @@ public void commitBatch() { @Override public void deleteBatch(Collection keys) { - if (!enabled) { - return; + lock.writeLock().lock(); + try { + if (!enabled) { + return; + } + // deletes are delayed + keys.forEach(key -> currentUpdates.deletedKeys.add(new ByteArrayWrapper(key))); + } catch (Exception e) { + if (e instanceof RuntimeException) { + throw e; + } else { + LOG.error("Could not delete batch due to ", e); + } + } finally { + lock.writeLock().unlock(); } - // deletes are delayed - keys.forEach(key -> currentUpdates.deletedKeys.add(new ByteArrayWrapper(key))); } @Override public boolean isEmpty() { - // the delayed deletes are not considered by this check until applied to the db - if (!currentUpdates.insertedKeys.isEmpty()) { - return false; - } else { - return src.isEmpty(); + lock.readLock().lock(); + try { + // the delayed deletes are not considered by this check until applied to the db + if (!currentUpdates.insertedKeys.isEmpty()) { + return false; + } else { + return src.isEmpty(); + } + } catch (Exception e) { + throw e; + } finally { + lock.readLock().unlock(); } } From 0fb29b90382fe8857ce993f45b12abd29bca2070 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Mon, 14 May 2018 11:46:53 -0400 Subject: [PATCH 03/35] disabling locking on the state database since the JournalPruneDS is locked --- modMcf/src/org/aion/mcf/db/AbstractRepository.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modMcf/src/org/aion/mcf/db/AbstractRepository.java b/modMcf/src/org/aion/mcf/db/AbstractRepository.java index 269727801b..45bc3429a1 100644 --- a/modMcf/src/org/aion/mcf/db/AbstractRepository.java +++ b/modMcf/src/org/aion/mcf/db/AbstractRepository.java @@ -182,8 +182,8 @@ protected void initializeDatabasesAndCaches() throws Exception { // getting state specific properties sharedProps = cfg.getDatabaseConfig(STATE_DB); - // locking enabled for state - sharedProps.setProperty(Props.ENABLE_LOCKING, "true"); + // locking enabled for state when JournalPrune not used + sharedProps.setProperty(Props.ENABLE_LOCKING, "false"); sharedProps.setProperty(Props.DB_PATH, cfg.getDbPath()); sharedProps.setProperty(Props.DB_NAME, STATE_DB); this.stateDatabase = connectAndOpen(sharedProps); From 8aa11337c3028cd5b871a33da946a24204be34c4 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Mon, 14 May 2018 11:50:24 -0400 Subject: [PATCH 04/35] moving the JournalPruneDS test class in the correct folder --- .../aion/{ => mcf}/trie/JournalPruneDataSourceTest.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) rename modMcf/test/org/aion/{ => mcf}/trie/JournalPruneDataSourceTest.java (99%) diff --git a/modMcf/test/org/aion/trie/JournalPruneDataSourceTest.java b/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java similarity index 99% rename from modMcf/test/org/aion/trie/JournalPruneDataSourceTest.java rename to modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java index 83106efb95..d90271035f 100644 --- a/modMcf/test/org/aion/trie/JournalPruneDataSourceTest.java +++ b/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java @@ -26,7 +26,7 @@ * Contributors to the aion source files in decreasing order of code volume: * Aion foundation. ******************************************************************************/ -package org.aion.trie; +package org.aion.mcf.trie; import static com.google.common.truth.Truth.assertThat; @@ -36,7 +36,6 @@ import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.db.impl.DatabaseFactory; import org.aion.log.AionLoggerFactory; -import org.aion.mcf.trie.JournalPruneDataSource; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; @@ -441,7 +440,7 @@ public void testDelete_wPrune() { } @Test - public void testDeleteBatchWithPrune() { + public void testDeleteBatch_wPrune() { // ensure existence Map map = new HashMap<>(); map.put(k1, v1); @@ -467,7 +466,7 @@ public void testDeleteBatchWithPrune() { } @Test - public void testKeys() { + public void testKeys_wPrune() { db.setPruneEnabled(true); // keys shouldn't be null even when empty From ca0c44bc8fc518fddade75feb046352144bd28e9 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Mon, 14 May 2018 13:55:39 -0400 Subject: [PATCH 05/35] unit tests for conformance to interface --- .../aion/mcf/trie/JournalPruneDataSource.java | 50 ++++++ .../mcf/trie/JournalPruneDataSourceTest.java | 155 +++++++++++++++++- 2 files changed, 202 insertions(+), 3 deletions(-) diff --git a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java index a2257a3d5c..a152be5908 100644 --- a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java +++ b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java @@ -98,6 +98,8 @@ public void setPruneEnabled(boolean e) { } public void put(byte[] key, byte[] value) { + checkNotNull(key); + lock.writeLock().lock(); try { @@ -116,6 +118,8 @@ public void put(byte[] key, byte[] value) { src.put(key, value); } else { + checkOpen(); + // Value does not exist, so we delete from current updates if (enabled) { currentUpdates.deletedKeys.add(keyW); @@ -134,9 +138,13 @@ public void put(byte[] key, byte[] value) { } public void delete(byte[] key) { + checkNotNull(key); + lock.writeLock().lock(); try { + checkOpen(); + if (!enabled) { return; } @@ -155,6 +163,8 @@ public void delete(byte[] key) { @Override public void putBatch(Map inputMap) { + checkNotNull(inputMap.keySet()); + lock.writeLock().lock(); try { @@ -343,8 +353,13 @@ public void commitBatch() { @Override public void deleteBatch(Collection keys) { + checkNotNull(keys); + lock.writeLock().lock(); + try { + checkOpen(); + if (!enabled) { return; } @@ -364,9 +379,11 @@ public void deleteBatch(Collection keys) { @Override public boolean isEmpty() { lock.readLock().lock(); + try { // the delayed deletes are not considered by this check until applied to the db if (!currentUpdates.insertedKeys.isEmpty()) { + checkOpen(); return false; } else { return src.isEmpty(); @@ -381,4 +398,37 @@ public boolean isEmpty() { public IByteArrayKeyValueDatabase getSrc() { return src; } + + /** + * Checks that the data store connection is open. Throws a {@link RuntimeException} if the data + * store connection is closed. + * + * @implNote Always do this check after acquiring a lock on the class/data. Otherwise it might + * produce inconsistent results due to lack of synchronization. + */ + protected void checkOpen() { + if (!src.isOpen()) { + throw new RuntimeException("Data store is not opened: " + src); + } + } + + /** + * Checks that the given key is not null. Throws a {@link IllegalArgumentException} if the key + * is null. + */ + public static void checkNotNull(byte[] k) { + if (k == null) { + throw new IllegalArgumentException("The data store does not accept null keys."); + } + } + + /** + * Checks that the given collection of keys does not contain null values. Throws a {@link + * IllegalArgumentException} if a null key is present. + */ + public static void checkNotNull(Collection keys) { + if (keys.contains(null)) { + throw new IllegalArgumentException("The data store does not accept null keys."); + } + } } diff --git a/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java b/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java index d90271035f..f92a3b7517 100644 --- a/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java +++ b/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java @@ -30,11 +30,10 @@ import static com.google.common.truth.Truth.assertThat; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; +import java.util.*; import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.db.impl.DatabaseFactory; +import org.aion.db.impl.DatabaseTestUtils; import org.aion.log.AionLoggerFactory; import org.junit.After; import org.junit.Before; @@ -78,6 +77,8 @@ public void close() { assertThat(source_db.isClosed()).isTrue(); } + // Pruning disabled tests ---------------------------------------------------- + @Test public void testPut_woPrune() { db.setPruneEnabled(false); @@ -314,6 +315,8 @@ public void testIsEmpty_woPrune() { assertThat(db.getDeletedKeysCount()).isEqualTo(0); } + // Pruning enabled tests ---------------------------------------------------- + @Test public void testPut_wPrune() { db.setPruneEnabled(true); @@ -549,4 +552,150 @@ public void testIsEmpty_wPrune() { assertThat(db.getInsertedKeysCount()).isEqualTo(3); assertThat(db.getDeletedKeysCount()).isEqualTo(3); } + + // Access with exception tests ---------------------------------------------------- + + @Test(expected = RuntimeException.class) + public void testIsEmpty_wClosedDatabase() { + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + // attempt isEmpty on closed db + db.isEmpty(); + } + + @Test(expected = RuntimeException.class) + public void testIsEmpty_wClosedDatabase_wInsertedKeys() { + db.put(DatabaseTestUtils.randomBytes(32), DatabaseTestUtils.randomBytes(32)); + + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + + // attempt isEmpty on closed db + db.isEmpty(); + } + + @Test(expected = RuntimeException.class) + public void testKeys_wClosedDatabase() { + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + // attempt keys on closed db + db.keys(); + } + + @Test(expected = RuntimeException.class) + public void testGet_wClosedDatabase() { + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + // attempt get on closed db + db.get(DatabaseTestUtils.randomBytes(32)); + } + + @Test(expected = RuntimeException.class) + public void testPut_wClosedDatabase() { + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + // attempt put on closed db + db.put(DatabaseTestUtils.randomBytes(32), DatabaseTestUtils.randomBytes(32)); + } + + @Test(expected = RuntimeException.class) + public void testPut_wClosedDatabase_wNullValue() { + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + // attempt put on closed db + db.put(DatabaseTestUtils.randomBytes(32), null); + } + + @Test(expected = RuntimeException.class) + public void testDelete_wClosedDatabase() { + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + // attempt delete on closed db + db.delete(DatabaseTestUtils.randomBytes(32)); + } + + @Test(expected = RuntimeException.class) + public void testPutBatch_wClosedDatabase() { + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + Map map = new HashMap<>(); + map.put(DatabaseTestUtils.randomBytes(32), DatabaseTestUtils.randomBytes(32)); + map.put(DatabaseTestUtils.randomBytes(32), DatabaseTestUtils.randomBytes(32)); + map.put(DatabaseTestUtils.randomBytes(32), DatabaseTestUtils.randomBytes(32)); + + // attempt putBatch on closed db + db.putBatch(map); + } + + @Test(expected = RuntimeException.class) + public void testDeleteBatch_wClosedDatabase() { + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + List list = new ArrayList<>(); + list.add(DatabaseTestUtils.randomBytes(32)); + list.add(DatabaseTestUtils.randomBytes(32)); + list.add(DatabaseTestUtils.randomBytes(32)); + + // attempt deleteBatch on closed db + db.deleteBatch(list); + } + + @Test(expected = IllegalArgumentException.class) + public void testGet_wNullKey() { + assertThat(source_db.open()).isTrue(); + + // attempt get with null key + db.get(null); + } + + @Test(expected = IllegalArgumentException.class) + public void testPut_wNullKey() { + assertThat(source_db.open()).isTrue(); + + // attempt put with null key + db.put(null, DatabaseTestUtils.randomBytes(32)); + } + + @Test(expected = IllegalArgumentException.class) + public void testDelete_wNullKey() { + assertThat(source_db.open()).isTrue(); + + // attempt delete with null key + db.delete(null); + } + + @Test(expected = IllegalArgumentException.class) + public void testPutBatch_wNullKey() { + assertThat(source_db.open()).isTrue(); + + Map map = new HashMap<>(); + map.put(DatabaseTestUtils.randomBytes(32), DatabaseTestUtils.randomBytes(32)); + map.put(DatabaseTestUtils.randomBytes(32), DatabaseTestUtils.randomBytes(32)); + map.put(null, DatabaseTestUtils.randomBytes(32)); + + // attempt putBatch on closed db + db.putBatch(map); + } + + @Test(expected = IllegalArgumentException.class) + public void testDeleteBatch_wNullKey() { + assertThat(source_db.open()).isTrue(); + + List list = new ArrayList<>(); + list.add(DatabaseTestUtils.randomBytes(32)); + list.add(DatabaseTestUtils.randomBytes(32)); + list.add(null); + + // attempt deleteBatch on closed db + db.deleteBatch(list); + } } From d10df6345a00e4f17232bdced842f9f497d4e3db Mon Sep 17 00:00:00 2001 From: Alexandra Date: Mon, 14 May 2018 14:04:36 -0400 Subject: [PATCH 06/35] concurrency tests for JournalPruneDS --- .../mcf/trie/JournalPruneDataSourceTest.java | 245 ++++++++++++++++++ 1 file changed, 245 insertions(+) diff --git a/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java b/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java index f92a3b7517..c318870db1 100644 --- a/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java +++ b/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java @@ -29,6 +29,7 @@ package org.aion.mcf.trie; import static com.google.common.truth.Truth.assertThat; +import static org.aion.db.impl.DatabaseTestUtils.assertConcurrent; import java.util.*; import org.aion.base.db.IByteArrayKeyValueDatabase; @@ -698,4 +699,248 @@ public void testDeleteBatch_wNullKey() { // attempt deleteBatch on closed db db.deleteBatch(list); } + + // Concurrent access tests ---------------------------------------------------- + + private static final int CONCURRENT_THREADS = 200; + private static final int TIME_OUT = 100; // in seconds + private static final boolean DISPLAY_MESSAGES = false; + + private static int count = 0; + + private static synchronized int getNext() { + count++; + return count; + } + + private void addThread_IsEmpty(List threads, JournalPruneDataSource db) { + threads.add( + () -> { + boolean check = db.isEmpty(); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + + ": " + + (check ? "EMPTY" : "NOT EMPTY")); + } + }); + } + + private void addThread_Keys(List threads, JournalPruneDataSource db) { + threads.add( + () -> { + Set keys = db.keys(); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + ": #keys = " + keys.size()); + } + }); + } + + private void addThread_Get(List threads, JournalPruneDataSource db, String key) { + threads.add( + () -> { + boolean hasValue = db.get(key.getBytes()).isPresent(); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + + ": " + + key + + " " + + (hasValue ? "PRESENT" : "NOT PRESENT")); + } + }); + } + + private void addThread_Put(List threads, JournalPruneDataSource db, String key) { + threads.add( + () -> { + db.put(key.getBytes(), DatabaseTestUtils.randomBytes(32)); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + ": " + key + " ADDED"); + } + }); + } + + private void addThread_Delete(List threads, JournalPruneDataSource db, String key) { + threads.add( + () -> { + db.delete(key.getBytes()); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + ": " + key + " DELETED"); + } + }); + } + + private void addThread_PutBatch(List threads, JournalPruneDataSource db, String key) { + threads.add( + () -> { + Map map = new HashMap<>(); + map.put((key + 1).getBytes(), DatabaseTestUtils.randomBytes(32)); + map.put((key + 2).getBytes(), DatabaseTestUtils.randomBytes(32)); + map.put((key + 3).getBytes(), DatabaseTestUtils.randomBytes(32)); + db.putBatch(map); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + + ": " + + (key + 1) + + ", " + + (key + 2) + + ", " + + (key + 3) + + " ADDED"); + } + }); + } + + private void addThread_DeleteBatch( + List threads, JournalPruneDataSource db, String key) { + threads.add( + () -> { + List list = new ArrayList<>(); + list.add((key + 1).getBytes()); + list.add((key + 2).getBytes()); + list.add((key + 3).getBytes()); + db.deleteBatch(list); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + + ": " + + (key + 1) + + ", " + + (key + 2) + + ", " + + (key + 3) + + " DELETED"); + } + }); + } + + @Test + public void testConcurrentAccessOnOpenDatabase() throws InterruptedException { + assertThat(source_db.isOpen()).isTrue(); + + // create distinct threads with + List threads = new ArrayList<>(); + + int threadSetCount = CONCURRENT_THREADS / 8; + if (threadSetCount < 3) { + threadSetCount = 3; + } + + for (int i = 0; i < threadSetCount; i++) { + // 1. thread that checks empty + addThread_IsEmpty(threads, db); + + // 2. thread that gets keys + addThread_Keys(threads, db); + + String keyStr = "key-" + i + "."; + + // 3. thread that gets entry + addThread_Get(threads, db, keyStr); + + // 4. thread that puts entry + addThread_Put(threads, db, keyStr); + + // 5. thread that deletes entry + addThread_Delete(threads, db, keyStr); + + // 6. thread that puts entries + addThread_PutBatch(threads, db, keyStr); + + // 7. thread that deletes entry + addThread_DeleteBatch(threads, db, keyStr); + } + + // run threads and check for exceptions + assertConcurrent("Testing concurrent access. ", threads, TIME_OUT); + + // ensuring close + db.close(); + assertThat(source_db.isClosed()).isTrue(); + } + + @Test + public void testConcurrentPut() throws InterruptedException { + assertThat(source_db.isOpen()).isTrue(); + + // create distinct threads with + List threads = new ArrayList<>(); + + for (int i = 0; i < CONCURRENT_THREADS; i++) { + addThread_Put(threads, db, "key-" + i); + } + + // run threads + assertConcurrent("Testing put(...) ", threads, TIME_OUT); + + // check that all values were added + assertThat(db.keys().size()).isEqualTo(CONCURRENT_THREADS); + + // ensuring close + db.close(); + assertThat(source_db.isClosed()).isTrue(); + } + + @Test + public void testConcurrentPutBatch() throws InterruptedException { + assertThat(source_db.isOpen()).isTrue(); + + // create distinct threads with + List threads = new ArrayList<>(); + + for (int i = 0; i < CONCURRENT_THREADS; i++) { + addThread_PutBatch(threads, db, "key-" + i); + } + + // run threads + assertConcurrent("Testing putBatch(...) ", threads, TIME_OUT); + + // check that all values were added + assertThat(db.keys().size()).isEqualTo(3 * CONCURRENT_THREADS); + + // ensuring close + db.close(); + assertThat(source_db.isClosed()).isTrue(); + } + + @Test + public void testConcurrentUpdate() throws InterruptedException { + assertThat(source_db.isOpen()).isTrue(); + + // create distinct threads with + List threads = new ArrayList<>(); + + int threadSetCount = CONCURRENT_THREADS / 4; + if (threadSetCount < 3) { + threadSetCount = 3; + } + + for (int i = 0; i < threadSetCount; i++) { + String keyStr = "key-" + i + "."; + + // 1. thread that puts entry + addThread_Put(threads, db, keyStr); + + // 2. thread that deletes entry + addThread_Delete(threads, db, keyStr); + + // 3. thread that puts entries + addThread_PutBatch(threads, db, keyStr); + + // 4. thread that deletes entry + addThread_DeleteBatch(threads, db, keyStr); + } + + // run threads and check for exceptions + assertConcurrent("Testing concurrent updates. ", threads, TIME_OUT); + + // ensuring close + db.close(); + assertThat(source_db.isClosed()).isTrue(); + } } From 3205e07e43e2f82c313d48ace2659436d9c2ca50 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Mon, 14 May 2018 16:25:32 -0400 Subject: [PATCH 07/35] unit tests for pruning functionality --- .../mcf/trie/JournalPruneDataSourceTest.java | 535 +++++++++++++++++- 1 file changed, 533 insertions(+), 2 deletions(-) diff --git a/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java b/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java index c318870db1..2d34a66ed8 100644 --- a/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java +++ b/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java @@ -318,6 +318,8 @@ public void testIsEmpty_woPrune() { // Pruning enabled tests ---------------------------------------------------- + private static final byte[] b0 = "block0".getBytes(); + @Test public void testPut_wPrune() { db.setPruneEnabled(true); @@ -332,6 +334,11 @@ public void testPut_wPrune() { // ensure the insert was propagated assertThat(source_db.get(k1).get()).isEqualTo(v1); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); } @Test @@ -356,6 +363,11 @@ public void testPutBatch_wPrune() { // ensure the inserts were propagated assertThat(source_db.get(k1).get()).isEqualTo(v1); assertThat(source_db.get(k2).get()).isEqualTo(v2); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); } @Test @@ -389,6 +401,11 @@ public void testUpdate_wPrune() { // ensure cached values assertThat(db.getDeletedKeysCount()).isEqualTo(2); assertThat(db.getInsertedKeysCount()).isEqualTo(2); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); } @Test @@ -423,6 +440,11 @@ public void testUpdateBatch_wPrune() { // ensure cached values assertThat(db.getDeletedKeysCount()).isEqualTo(1); assertThat(db.getInsertedKeysCount()).isEqualTo(3); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); } @Test @@ -441,6 +463,11 @@ public void testDelete_wPrune() { // ensure cached values assertThat(db.getDeletedKeysCount()).isEqualTo(1); assertThat(db.getInsertedKeysCount()).isEqualTo(1); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); } @Test @@ -467,6 +494,11 @@ public void testDeleteBatch_wPrune() { // ensure cached values assertThat(db.getDeletedKeysCount()).isEqualTo(3); assertThat(db.getInsertedKeysCount()).isEqualTo(2); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); } @Test @@ -514,6 +546,11 @@ public void testKeys_wPrune() { // ensure no cached values assertThat(db.getInsertedKeysCount()).isEqualTo(3); assertThat(db.getDeletedKeysCount()).isEqualTo(3); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); } @Test @@ -552,6 +589,11 @@ public void testIsEmpty_wPrune() { // ensure no cached values assertThat(db.getInsertedKeysCount()).isEqualTo(3); assertThat(db.getDeletedKeysCount()).isEqualTo(3); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); } // Access with exception tests ---------------------------------------------------- @@ -819,9 +861,44 @@ private void addThread_DeleteBatch( }); } + private void addThread_StoreBlockChanges( + List threads, JournalPruneDataSource db, String hash, long number) { + threads.add( + () -> { + db.storeBlockChanges(hash.getBytes(), number); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + + ": block (" + + hash + + ", " + + number + + ") STORED"); + } + }); + } + + private void addThread_Prune( + List threads, JournalPruneDataSource db, String hash, long number) { + threads.add( + () -> { + db.prune(hash.getBytes(), number); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + + ": block (" + + hash + + ", " + + number + + ") PRUNED"); + } + }); + } + @Test public void testConcurrentAccessOnOpenDatabase() throws InterruptedException { assertThat(source_db.isOpen()).isTrue(); + db.setPruneEnabled(true); // create distinct threads with List threads = new ArrayList<>(); @@ -831,6 +908,8 @@ public void testConcurrentAccessOnOpenDatabase() throws InterruptedException { threadSetCount = 3; } + String keyStr, blockStr; + for (int i = 0; i < threadSetCount; i++) { // 1. thread that checks empty addThread_IsEmpty(threads, db); @@ -838,7 +917,7 @@ public void testConcurrentAccessOnOpenDatabase() throws InterruptedException { // 2. thread that gets keys addThread_Keys(threads, db); - String keyStr = "key-" + i + "."; + keyStr = "key-" + i + "."; // 3. thread that gets entry addThread_Get(threads, db, keyStr); @@ -854,6 +933,14 @@ public void testConcurrentAccessOnOpenDatabase() throws InterruptedException { // 7. thread that deletes entry addThread_DeleteBatch(threads, db, keyStr); + + blockStr = "block-" + i + "."; + + // 8. thread that stores block changes + addThread_StoreBlockChanges(threads, db, blockStr, i); + + // 9. thread that prunes + addThread_Prune(threads, db, blockStr, i); } // run threads and check for exceptions @@ -867,6 +954,7 @@ public void testConcurrentAccessOnOpenDatabase() throws InterruptedException { @Test public void testConcurrentPut() throws InterruptedException { assertThat(source_db.isOpen()).isTrue(); + db.setPruneEnabled(true); // create distinct threads with List threads = new ArrayList<>(); @@ -889,6 +977,7 @@ public void testConcurrentPut() throws InterruptedException { @Test public void testConcurrentPutBatch() throws InterruptedException { assertThat(source_db.isOpen()).isTrue(); + db.setPruneEnabled(true); // create distinct threads with List threads = new ArrayList<>(); @@ -911,6 +1000,7 @@ public void testConcurrentPutBatch() throws InterruptedException { @Test public void testConcurrentUpdate() throws InterruptedException { assertThat(source_db.isOpen()).isTrue(); + db.setPruneEnabled(true); // create distinct threads with List threads = new ArrayList<>(); @@ -920,8 +1010,10 @@ public void testConcurrentUpdate() throws InterruptedException { threadSetCount = 3; } + String keyStr, blockStr; + for (int i = 0; i < threadSetCount; i++) { - String keyStr = "key-" + i + "."; + keyStr = "key-" + i + "."; // 1. thread that puts entry addThread_Put(threads, db, keyStr); @@ -934,6 +1026,14 @@ public void testConcurrentUpdate() throws InterruptedException { // 4. thread that deletes entry addThread_DeleteBatch(threads, db, keyStr); + + blockStr = "block-" + i + "."; + + // 5. thread that stores block changes + addThread_StoreBlockChanges(threads, db, blockStr, i); + + // 6. thread that prunes + addThread_Prune(threads, db, blockStr, i); } // run threads and check for exceptions @@ -943,4 +1043,435 @@ public void testConcurrentUpdate() throws InterruptedException { db.close(); assertThat(source_db.isClosed()).isTrue(); } + + // Pruning tests ---------------------------------------------------- + + private static final byte[] b1 = "block1".getBytes(); + private static final byte[] b2 = "block2".getBytes(); + private static final byte[] b3 = "block3".getBytes(); + + private static final byte[] k4 = "key4".getBytes(); + private static final byte[] v4 = "value4".getBytes(); + + private static final byte[] k5 = "key5".getBytes(); + private static final byte[] v5 = "value5".getBytes(); + + private static final byte[] k6 = "key6".getBytes(); + private static final byte[] v6 = "value6".getBytes(); + + @Test + public void pruningTest() { + db.setPruneEnabled(true); + + // block 0 + db.put(k1, v1); + db.put(k2, v2); + db.put(k3, v3); + assertThat(db.getInsertedKeysCount()).isEqualTo(3); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + db.storeBlockChanges(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // block 1 + db.put(k4, v4); + db.delete(k2); + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.get(k2).isPresent()).isTrue(); + assertThat(source_db.get(k4).get()).isEqualTo(v4); + + // block 2 + db.put(k2, v3); + db.delete(k3); + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b2, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).isPresent()).isTrue(); + + // block 3 + db.put(k5, v5); + db.put(k6, v6); + db.delete(k2); + assertThat(db.getInsertedKeysCount()).isEqualTo(2); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b3, 3); + assertThat(db.getBlockUpdates().size()).isEqualTo(4); + + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + assertThat(source_db.get(k2).isPresent()).isTrue(); + + // prune block 0 + db.prune(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // prune block 1 + db.prune(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.get(k4).get()).isEqualTo(v4); + // not deleted due to block 2 insert + assertThat(source_db.get(k2).get()).isEqualTo(v3); + + // prune block 2 + db.prune(b2, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).isPresent()).isFalse(); + + // prune block 3 + db.prune(b3, 3); + assertThat(db.getBlockUpdates().size()).isEqualTo(0); + + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + assertThat(source_db.get(k2).isPresent()).isFalse(); + } + + @Test + public void pruningTest_wBatch() { + db.setPruneEnabled(true); + + Map map = new HashMap<>(); + + // block 0 + map.put(k1, v1); + map.put(k2, v2); + map.put(k3, v3); + db.putBatch(map); + + assertThat(db.getInsertedKeysCount()).isEqualTo(3); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + db.storeBlockChanges(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // block 1 + map.clear(); + map.put(k4, v4); + map.put(k2, null); + db.putBatch(map); + + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.get(k2).isPresent()).isTrue(); + assertThat(source_db.get(k4).get()).isEqualTo(v4); + + // block 2 + map.clear(); + map.put(k2, v3); + map.put(k3, null); + db.putBatch(map); + + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b2, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).isPresent()).isTrue(); + + // block 3 + map.clear(); + map.put(k5, v5); + map.put(k6, v6); + map.put(k2, null); + db.putBatch(map); + + assertThat(db.getInsertedKeysCount()).isEqualTo(2); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b3, 3); + assertThat(db.getBlockUpdates().size()).isEqualTo(4); + + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + assertThat(source_db.get(k2).isPresent()).isTrue(); + + // prune block 0 + db.prune(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // prune block 1 + db.prune(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.get(k4).get()).isEqualTo(v4); + // not deleted due to block 2 insert + assertThat(source_db.get(k2).get()).isEqualTo(v3); + + // prune block 2 + db.prune(b2, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).isPresent()).isFalse(); + + // prune block 3 + db.prune(b3, 3); + assertThat(db.getBlockUpdates().size()).isEqualTo(0); + + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + assertThat(source_db.get(k2).isPresent()).isFalse(); + } + + @Test + public void pruningTest_woStoredLevel() { + + source_db.put(k2, v2); + source_db.put(k3, v3); + + // block 2 + db.put(k2, v3); + db.delete(k3); + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b2, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).isPresent()).isTrue(); + + // block 3 + db.put(k5, v5); + db.put(k6, v6); + db.delete(k2); + assertThat(db.getInsertedKeysCount()).isEqualTo(2); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b3, 3); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + assertThat(source_db.get(k2).isPresent()).isTrue(); + + // prune block 0 (not stored) + db.prune(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + // prune block 1 (not stored) + db.prune(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + + // prune block 2 + db.prune(b2, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).isPresent()).isFalse(); + + // prune block 3 + db.prune(b3, 3); + assertThat(db.getBlockUpdates().size()).isEqualTo(0); + + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + assertThat(source_db.get(k2).isPresent()).isFalse(); + } + + @Test + public void pruningTest_wFork_onCurrentLevel() { + db.setPruneEnabled(true); + + // block b0 + db.put(k1, v1); + db.put(k2, v2); + db.put(k3, v3); + db.storeBlockChanges(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.keys().size()).isEqualTo(3); + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // block b1 + db.put(k4, v4); + db.put(k1, v2); + db.delete(k2); + db.storeBlockChanges(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.keys().size()).isEqualTo(4); + assertThat(source_db.get(k1).get()).isEqualTo(v2); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k4).get()).isEqualTo(v4); + + // block b2 + db.put(k5, v5); + db.delete(k3); + db.put(k2, v3); + db.put(k1, v4); + db.put(k4, v6); + db.storeBlockChanges(b2, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + + assertThat(source_db.keys().size()).isEqualTo(5); + assertThat(source_db.get(k1).get()).isEqualTo(v4); + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k4).get()).isEqualTo(v6); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + + // block b3 : note same level as block b2 + db.put(k6, v6); + db.delete(k4); + db.put(k2, v4); + db.put(k1, v3); + db.storeBlockChanges(b3, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(4); + + assertThat(source_db.keys().size()).isEqualTo(6); + assertThat(source_db.get(k1).get()).isEqualTo(v3); + assertThat(source_db.get(k2).get()).isEqualTo(v4); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k4).get()).isEqualTo(v6); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + + // prune block b0 + db.prune(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + assertThat(source_db.keys().size()).isEqualTo(6); + + // prune block b1 + db.prune(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + assertThat(source_db.keys().size()).isEqualTo(6); + + // prune block b3 at level 2 (should be called for main chain block) + db.prune(b3, 2); + // also removed the updates for block b2 + assertThat(db.getBlockUpdates().size()).isEqualTo(0); + + assertThat(source_db.keys().size()).isEqualTo(5); + assertThat(source_db.get(k1).get()).isEqualTo(v3); + assertThat(source_db.get(k2).get()).isEqualTo(v4); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k4).isPresent()).isFalse(); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + } + + @Test + public void pruningTest_wFork_onPastLevel() { + db.setPruneEnabled(true); + + // block b0 + db.put(k1, v1); + db.put(k2, v2); + db.put(k3, v3); + db.storeBlockChanges(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.keys().size()).isEqualTo(3); + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // block b1 + db.put(k4, v4); + db.put(k1, v2); + db.delete(k2); + db.storeBlockChanges(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.keys().size()).isEqualTo(4); + assertThat(source_db.get(k1).get()).isEqualTo(v2); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k4).get()).isEqualTo(v4); + + // block b2 : note same level as block b1 + db.put(k5, v5); + db.delete(k3); + db.put(k2, v3); + db.put(k1, v4); + db.storeBlockChanges(b2, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + + assertThat(source_db.keys().size()).isEqualTo(5); + assertThat(source_db.get(k1).get()).isEqualTo(v4); + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k4).get()).isEqualTo(v4); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + + // block b3 + db.put(k6, v6); + db.delete(k4); + db.put(k2, v4); + db.put(k1, v3); + db.storeBlockChanges(b3, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(4); + + assertThat(source_db.keys().size()).isEqualTo(6); + assertThat(source_db.get(k1).get()).isEqualTo(v3); + assertThat(source_db.get(k2).get()).isEqualTo(v4); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k4).get()).isEqualTo(v4); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + + // prune block b0 + db.prune(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + assertThat(source_db.keys().size()).isEqualTo(6); + + // prune block b2 at level 1 : (should be called for main chain block) + db.prune(b2, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + assertThat(source_db.keys().size()).isEqualTo(5); + assertThat(source_db.get(k1).get()).isEqualTo(v3); + assertThat(source_db.get(k2).get()).isEqualTo(v4); + assertThat(source_db.get(k3).isPresent()).isFalse(); + // note: k4 must be explicitly deleted even if it was introduced in the discarded b1 block + assertThat(source_db.get(k4).get()).isEqualTo(v4); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + + // prune block b3 + db.prune(b3, 2); + // also removed the updates for block b2 + assertThat(db.getBlockUpdates().size()).isEqualTo(0); + + assertThat(source_db.keys().size()).isEqualTo(4); + assertThat(source_db.get(k1).get()).isEqualTo(v3); + assertThat(source_db.get(k2).get()).isEqualTo(v4); + assertThat(source_db.get(k3).isPresent()).isFalse(); + assertThat(source_db.get(k4).isPresent()).isFalse(); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + } } From 0f78cc808652529e3580569d9ba04503fb8aa28f Mon Sep 17 00:00:00 2001 From: Alexandra Date: Mon, 14 May 2018 16:42:29 -0400 Subject: [PATCH 08/35] removed dependency on modDbImpl --- .../mcf/trie/JournalPruneDataSourceTest.java | 115 +++++++++++++----- 1 file changed, 85 insertions(+), 30 deletions(-) diff --git a/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java b/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java index 2d34a66ed8..bd7c41153c 100644 --- a/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java +++ b/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java @@ -29,12 +29,15 @@ package org.aion.mcf.trie; import static com.google.common.truth.Truth.assertThat; -import static org.aion.db.impl.DatabaseTestUtils.assertConcurrent; +import static org.junit.Assert.assertTrue; import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.db.impl.DatabaseFactory; -import org.aion.db.impl.DatabaseTestUtils; import org.aion.log.AionLoggerFactory; import org.junit.After; import org.junit.Before; @@ -609,7 +612,7 @@ public void testIsEmpty_wClosedDatabase() { @Test(expected = RuntimeException.class) public void testIsEmpty_wClosedDatabase_wInsertedKeys() { - db.put(DatabaseTestUtils.randomBytes(32), DatabaseTestUtils.randomBytes(32)); + db.put(randomBytes(32), randomBytes(32)); source_db.close(); assertThat(source_db.isOpen()).isFalse(); @@ -634,7 +637,7 @@ public void testGet_wClosedDatabase() { assertThat(source_db.isOpen()).isFalse(); // attempt get on closed db - db.get(DatabaseTestUtils.randomBytes(32)); + db.get(randomBytes(32)); } @Test(expected = RuntimeException.class) @@ -643,7 +646,7 @@ public void testPut_wClosedDatabase() { assertThat(source_db.isOpen()).isFalse(); // attempt put on closed db - db.put(DatabaseTestUtils.randomBytes(32), DatabaseTestUtils.randomBytes(32)); + db.put(randomBytes(32), randomBytes(32)); } @Test(expected = RuntimeException.class) @@ -652,7 +655,7 @@ public void testPut_wClosedDatabase_wNullValue() { assertThat(source_db.isOpen()).isFalse(); // attempt put on closed db - db.put(DatabaseTestUtils.randomBytes(32), null); + db.put(randomBytes(32), null); } @Test(expected = RuntimeException.class) @@ -661,7 +664,7 @@ public void testDelete_wClosedDatabase() { assertThat(source_db.isOpen()).isFalse(); // attempt delete on closed db - db.delete(DatabaseTestUtils.randomBytes(32)); + db.delete(randomBytes(32)); } @Test(expected = RuntimeException.class) @@ -670,9 +673,9 @@ public void testPutBatch_wClosedDatabase() { assertThat(source_db.isOpen()).isFalse(); Map map = new HashMap<>(); - map.put(DatabaseTestUtils.randomBytes(32), DatabaseTestUtils.randomBytes(32)); - map.put(DatabaseTestUtils.randomBytes(32), DatabaseTestUtils.randomBytes(32)); - map.put(DatabaseTestUtils.randomBytes(32), DatabaseTestUtils.randomBytes(32)); + map.put(randomBytes(32), randomBytes(32)); + map.put(randomBytes(32), randomBytes(32)); + map.put(randomBytes(32), randomBytes(32)); // attempt putBatch on closed db db.putBatch(map); @@ -684,9 +687,9 @@ public void testDeleteBatch_wClosedDatabase() { assertThat(source_db.isOpen()).isFalse(); List list = new ArrayList<>(); - list.add(DatabaseTestUtils.randomBytes(32)); - list.add(DatabaseTestUtils.randomBytes(32)); - list.add(DatabaseTestUtils.randomBytes(32)); + list.add(randomBytes(32)); + list.add(randomBytes(32)); + list.add(randomBytes(32)); // attempt deleteBatch on closed db db.deleteBatch(list); @@ -705,7 +708,7 @@ public void testPut_wNullKey() { assertThat(source_db.open()).isTrue(); // attempt put with null key - db.put(null, DatabaseTestUtils.randomBytes(32)); + db.put(null, randomBytes(32)); } @Test(expected = IllegalArgumentException.class) @@ -721,9 +724,9 @@ public void testPutBatch_wNullKey() { assertThat(source_db.open()).isTrue(); Map map = new HashMap<>(); - map.put(DatabaseTestUtils.randomBytes(32), DatabaseTestUtils.randomBytes(32)); - map.put(DatabaseTestUtils.randomBytes(32), DatabaseTestUtils.randomBytes(32)); - map.put(null, DatabaseTestUtils.randomBytes(32)); + map.put(randomBytes(32), randomBytes(32)); + map.put(randomBytes(32), randomBytes(32)); + map.put(null, randomBytes(32)); // attempt putBatch on closed db db.putBatch(map); @@ -734,27 +737,26 @@ public void testDeleteBatch_wNullKey() { assertThat(source_db.open()).isTrue(); List list = new ArrayList<>(); - list.add(DatabaseTestUtils.randomBytes(32)); - list.add(DatabaseTestUtils.randomBytes(32)); + list.add(randomBytes(32)); + list.add(randomBytes(32)); list.add(null); // attempt deleteBatch on closed db db.deleteBatch(list); } + public static byte[] randomBytes(int length) { + byte[] result = new byte[length]; + new Random().nextBytes(result); + return result; + } + // Concurrent access tests ---------------------------------------------------- private static final int CONCURRENT_THREADS = 200; private static final int TIME_OUT = 100; // in seconds private static final boolean DISPLAY_MESSAGES = false; - private static int count = 0; - - private static synchronized int getNext() { - count++; - return count; - } - private void addThread_IsEmpty(List threads, JournalPruneDataSource db) { threads.add( () -> { @@ -797,7 +799,7 @@ private void addThread_Get(List threads, JournalPruneDataSource db, St private void addThread_Put(List threads, JournalPruneDataSource db, String key) { threads.add( () -> { - db.put(key.getBytes(), DatabaseTestUtils.randomBytes(32)); + db.put(key.getBytes(), randomBytes(32)); if (DISPLAY_MESSAGES) { System.out.println( Thread.currentThread().getName() + ": " + key + " ADDED"); @@ -820,9 +822,9 @@ private void addThread_PutBatch(List threads, JournalPruneDataSource d threads.add( () -> { Map map = new HashMap<>(); - map.put((key + 1).getBytes(), DatabaseTestUtils.randomBytes(32)); - map.put((key + 2).getBytes(), DatabaseTestUtils.randomBytes(32)); - map.put((key + 3).getBytes(), DatabaseTestUtils.randomBytes(32)); + map.put((key + 1).getBytes(), randomBytes(32)); + map.put((key + 2).getBytes(), randomBytes(32)); + map.put((key + 3).getBytes(), randomBytes(32)); db.putBatch(map); if (DISPLAY_MESSAGES) { System.out.println( @@ -1044,6 +1046,59 @@ public void testConcurrentUpdate() throws InterruptedException { assertThat(source_db.isClosed()).isTrue(); } + /** + * From JUnit + * Wiki on multithreaded code and concurrency + */ + public static void assertConcurrent( + final String message, + final List runnables, + final int maxTimeoutSeconds) + throws InterruptedException { + final int numThreads = runnables.size(); + final List exceptions = Collections.synchronizedList(new ArrayList()); + final ExecutorService threadPool = Executors.newFixedThreadPool(numThreads); + try { + final CountDownLatch allExecutorThreadsReady = new CountDownLatch(numThreads); + final CountDownLatch afterInitBlocker = new CountDownLatch(1); + final CountDownLatch allDone = new CountDownLatch(numThreads); + for (final Runnable submittedTestRunnable : runnables) { + threadPool.submit( + () -> { + allExecutorThreadsReady.countDown(); + try { + afterInitBlocker.await(); + submittedTestRunnable.run(); + } catch (final Throwable e) { + exceptions.add(e); + } finally { + allDone.countDown(); + } + }); + } + // wait until all threads are ready + assertTrue( + "Timeout initializing threads! Perform long lasting initializations before passing runnables to assertConcurrent", + allExecutorThreadsReady.await(runnables.size() * 10, TimeUnit.MILLISECONDS)); + // start all test runners + afterInitBlocker.countDown(); + assertTrue( + message + " timeout! More than" + maxTimeoutSeconds + "seconds", + allDone.await(maxTimeoutSeconds, TimeUnit.SECONDS)); + } finally { + threadPool.shutdownNow(); + } + if (!exceptions.isEmpty()) { + for (Throwable e : exceptions) { + e.printStackTrace(); + } + } + assertTrue( + message + "failed with " + exceptions.size() + " exception(s):" + exceptions, + exceptions.isEmpty()); + } + // Pruning tests ---------------------------------------------------- private static final byte[] b1 = "block1".getBytes(); From 527f3820cf088da5012230bef072e75c86b0c317 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Thu, 17 May 2018 11:07:23 -0400 Subject: [PATCH 09/35] applying review comments --- .../aion/mcf/trie/JournalPruneDataSource.java | 100 ++++++++++-------- 1 file changed, 55 insertions(+), 45 deletions(-) diff --git a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java index a152be5908..135628a801 100644 --- a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java +++ b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java @@ -35,6 +35,7 @@ package org.aion.mcf.trie; import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.aion.base.db.IByteArrayKeyValueDatabase; @@ -83,18 +84,14 @@ public int getTotRefs() { // block hash => updates private LinkedHashMap blockUpdates = new LinkedHashMap<>(); private Updates currentUpdates = new Updates(); - private boolean enabled = true; + private AtomicBoolean enabled = new AtomicBoolean(true); public JournalPruneDataSource(IByteArrayKeyValueDatabase src) { this.src = src; } - public void setPruneEnabled(boolean e) { - lock.writeLock().lock(); - - enabled = e; - - lock.writeLock().unlock(); + public void setPruneEnabled(boolean _enabled) { + enabled.set(_enabled); } public void put(byte[] key, byte[] value) { @@ -103,28 +100,32 @@ public void put(byte[] key, byte[] value) { lock.writeLock().lock(); try { - ByteArrayWrapper keyW = new ByteArrayWrapper(key); - - // Check to see the value exists. - if (value != null) { + if (enabled.get()) { + // pruning enabled + ByteArrayWrapper keyW = ByteArrayWrapper.wrap(key); - // If it exists and pruning is enabled. - if (enabled) { + // Check to see the value exists. + if (value != null) { + // If it exists and pruning is enabled. currentUpdates.insertedKeys.add(keyW); incRef(keyW); - } - // put to source database. - src.put(key, value); + // put to source database. + src.put(key, value); - } else { - checkOpen(); + } else { + checkOpen(); - // Value does not exist, so we delete from current updates - if (enabled) { + // Value does not exist, so we delete from current updates currentUpdates.deletedKeys.add(keyW); } - // delete is not sent to source db + } else { + // pruning disabled + if (value != null) { + src.put(key, value); + } else { + checkOpen(); + } } } catch (Exception e) { if (e instanceof RuntimeException) { @@ -138,6 +139,9 @@ public void put(byte[] key, byte[] value) { } public void delete(byte[] key) { + if (!enabled.get()) { + return; + } checkNotNull(key); lock.writeLock().lock(); @@ -145,10 +149,7 @@ public void delete(byte[] key) { try { checkOpen(); - if (!enabled) { - return; - } - currentUpdates.deletedKeys.add(new ByteArrayWrapper(key)); + currentUpdates.deletedKeys.add(ByteArrayWrapper.wrap(key)); // delete is delayed } catch (Exception e) { if (e instanceof RuntimeException) { @@ -169,19 +170,23 @@ public void putBatch(Map inputMap) { try { Map insertsOnly = new HashMap<>(); - for (Map.Entry entry : inputMap.entrySet()) { - ByteArrayWrapper keyW = new ByteArrayWrapper(entry.getKey()); - if (entry.getValue() != null) { - if (enabled) { + if (enabled.get()) { + for (Map.Entry entry : inputMap.entrySet()) { + ByteArrayWrapper keyW = ByteArrayWrapper.wrap(entry.getKey()); + if (entry.getValue() != null) { currentUpdates.insertedKeys.add(keyW); incRef(keyW); - } - insertsOnly.put(entry.getKey(), entry.getValue()); - } else { - if (enabled) { + insertsOnly.put(entry.getKey(), entry.getValue()); + } else { currentUpdates.deletedKeys.add(keyW); } } + } else { + for (Map.Entry entry : inputMap.entrySet()) { + if (entry.getValue() != null) { + insertsOnly.put(entry.getKey(), entry.getValue()); + } + } } src.putBatch(insertsOnly); } catch (Exception e) { @@ -214,13 +219,14 @@ private Ref decRef(ByteArrayWrapper keyW) { } public void storeBlockChanges(byte[] blockHash, long blockNumber) { + if (!enabled.get()) { + return; + } + lock.writeLock().lock(); try { - if (!enabled) { - return; - } - ByteArrayWrapper hash = new ByteArrayWrapper(blockHash); + ByteArrayWrapper hash = ByteArrayWrapper.wrap(blockHash); currentUpdates.blockHeader = hash; currentUpdates.blockNumber = blockNumber; blockUpdates.put(hash, currentUpdates); @@ -231,13 +237,14 @@ public void storeBlockChanges(byte[] blockHash, long blockNumber) { } public void prune(byte[] blockHash, long blockNumber) { + if (!enabled.get()) { + return; + } + lock.writeLock().lock(); try { - if (!enabled) { - return; - } - ByteArrayWrapper blockHashW = new ByteArrayWrapper(blockHash); + ByteArrayWrapper blockHashW = ByteArrayWrapper.wrap(blockHash); Updates updates = blockUpdates.remove(blockHashW); if (updates != null) { for (ByteArrayWrapper insertedKey : updates.insertedKeys) { @@ -313,6 +320,7 @@ public Optional get(byte[] key) { try { return src.get(key); } catch (Exception e) { + LOG.error("Could not get key due to ", e); throw e; } finally { lock.readLock().unlock(); @@ -324,6 +332,7 @@ public Set keys() { try { return src.keys(); } catch (Exception e) { + LOG.error("Could not get keys due to ", e); throw e; } finally { lock.readLock().unlock(); @@ -353,6 +362,9 @@ public void commitBatch() { @Override public void deleteBatch(Collection keys) { + if (!enabled.get()) { + return; + } checkNotNull(keys); lock.writeLock().lock(); @@ -360,11 +372,8 @@ public void deleteBatch(Collection keys) { try { checkOpen(); - if (!enabled) { - return; - } // deletes are delayed - keys.forEach(key -> currentUpdates.deletedKeys.add(new ByteArrayWrapper(key))); + keys.forEach(key -> currentUpdates.deletedKeys.add(ByteArrayWrapper.wrap(key))); } catch (Exception e) { if (e instanceof RuntimeException) { throw e; @@ -389,6 +398,7 @@ public boolean isEmpty() { return src.isEmpty(); } } catch (Exception e) { + LOG.error("Could not check if empty due to ", e); throw e; } finally { lock.readLock().unlock(); From 606b1e3e9b29619b20c6515e85dcd7b78693976e Mon Sep 17 00:00:00 2001 From: Alexandra Date: Tue, 22 May 2018 11:16:04 -0400 Subject: [PATCH 10/35] bugfix: incorrect check for presence in the data source --- .../org/aion/mcf/trie/JournalPruneDataSource.java | 13 +++++++++---- .../aion/mcf/trie/JournalPruneDataSourceTest.java | 10 ++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java index 135628a801..31a1a1e9a1 100644 --- a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java +++ b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java @@ -76,6 +76,11 @@ public Ref(boolean dbRef) { public int getTotRefs() { return journalRefs + (dbRef ? 1 : 0); } + + @Override + public String toString() { + return "refs: " + String.valueOf(journalRefs) + " db: " + String.valueOf(dbRef); + } } Map refCount = new HashMap<>(); @@ -203,7 +208,7 @@ public void putBatch(Map inputMap) { private void incRef(ByteArrayWrapper keyW) { Ref cnt = refCount.get(keyW); if (cnt == null) { - cnt = new Ref(src.get(keyW.getData()) != null); + cnt = new Ref(src.get(keyW.getData()).isPresent()); refCount.put(keyW, cnt); } cnt.journalRefs++; @@ -279,14 +284,14 @@ private void rollbackForkBlocks(long blockNum) { private void rollback(ByteArrayWrapper blockHashW) { Updates updates = blockUpdates.remove(blockHashW); - Map batchRemove = new HashMap<>(); + List batchRemove = new ArrayList<>(); for (ByteArrayWrapper insertedKey : updates.insertedKeys) { Ref ref = decRef(insertedKey); if (ref.getTotRefs() == 0) { - batchRemove.put(insertedKey.getData(), null); + batchRemove.add(insertedKey.getData()); } } - src.putBatch(batchRemove); + src.deleteBatch(batchRemove); } public Map getRefCount() { diff --git a/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java b/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java index bd7c41153c..6e0dab3af0 100644 --- a/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java +++ b/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java @@ -1430,12 +1430,12 @@ public void pruningTest_wFork_onCurrentLevel() { // also removed the updates for block b2 assertThat(db.getBlockUpdates().size()).isEqualTo(0); - assertThat(source_db.keys().size()).isEqualTo(5); + assertThat(source_db.keys().size()).isEqualTo(4); assertThat(source_db.get(k1).get()).isEqualTo(v3); assertThat(source_db.get(k2).get()).isEqualTo(v4); assertThat(source_db.get(k3).get()).isEqualTo(v3); assertThat(source_db.get(k4).isPresent()).isFalse(); - assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k5).isPresent()).isFalse(); assertThat(source_db.get(k6).get()).isEqualTo(v6); } @@ -1485,7 +1485,6 @@ public void pruningTest_wFork_onPastLevel() { // block b3 db.put(k6, v6); - db.delete(k4); db.put(k2, v4); db.put(k1, v3); db.storeBlockChanges(b3, 2); @@ -1507,12 +1506,11 @@ public void pruningTest_wFork_onPastLevel() { // prune block b2 at level 1 : (should be called for main chain block) db.prune(b2, 1); assertThat(db.getBlockUpdates().size()).isEqualTo(1); - assertThat(source_db.keys().size()).isEqualTo(5); + assertThat(source_db.keys().size()).isEqualTo(4); assertThat(source_db.get(k1).get()).isEqualTo(v3); assertThat(source_db.get(k2).get()).isEqualTo(v4); assertThat(source_db.get(k3).isPresent()).isFalse(); - // note: k4 must be explicitly deleted even if it was introduced in the discarded b1 block - assertThat(source_db.get(k4).get()).isEqualTo(v4); + assertThat(source_db.get(k4).isPresent()).isFalse(); assertThat(source_db.get(k5).get()).isEqualTo(v5); assertThat(source_db.get(k6).get()).isEqualTo(v6); From d7c1f587e926f1104c7caf3f568d5685cd10f88d Mon Sep 17 00:00:00 2001 From: Alexandra Date: Tue, 15 May 2018 13:44:28 -0400 Subject: [PATCH 11/35] new configuration class and interface for pruning --- .../src/org/aion/base/db/IPruneConfig.java | 30 ++++ modMcf/src/org/aion/mcf/config/CfgPrune.java | 139 ++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 modAionBase/src/org/aion/base/db/IPruneConfig.java create mode 100644 modMcf/src/org/aion/mcf/config/CfgPrune.java diff --git a/modAionBase/src/org/aion/base/db/IPruneConfig.java b/modAionBase/src/org/aion/base/db/IPruneConfig.java new file mode 100644 index 0000000000..871537b85a --- /dev/null +++ b/modAionBase/src/org/aion/base/db/IPruneConfig.java @@ -0,0 +1,30 @@ +package org.aion.base.db; + +/** + * Interface for pruning configuration parameters. + * + * @author Alexandra Roatis + */ +public interface IPruneConfig { + + /** + * Indicates if pruning should be enabled or disabled. + * + * @return {@code true} when pruning enabled, {@code false} when pruning disabled. + */ + boolean isEnabled(); + + /** + * @return the number of topmost blocks for which the full data should be maintained on disk. + */ + int getCurrentCount(); + + /** + * Gets the rate at which blocks should be archived (for which the full data should be + * maintained on disk). Blocks that are exact multiples of the returned value should be + * persisted on disk, regardless of other pruning. + * + * @return integer value representing the archive rate + */ + int getArchiveRate(); +} diff --git a/modMcf/src/org/aion/mcf/config/CfgPrune.java b/modMcf/src/org/aion/mcf/config/CfgPrune.java new file mode 100644 index 0000000000..4e71fed6e9 --- /dev/null +++ b/modMcf/src/org/aion/mcf/config/CfgPrune.java @@ -0,0 +1,139 @@ +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: + * Aion foundation. + ******************************************************************************/ +package org.aion.mcf.config; + +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import javax.xml.stream.XMLStreamWriter; +import org.aion.base.db.IPruneConfig; + +/** + * Configuration for data pruning behavior. + * + * @author Alexandra Roatis + */ +public class CfgPrune implements IPruneConfig { + + private boolean enabled; + private int current; + private int archive; + + public CfgPrune() { + this.enabled = true; + this.current = 128; + this.archive = 10000; + } + + public CfgPrune(boolean _enabled) { + this.enabled = _enabled; + this.current = 128; + this.archive = 10000; + } + + public void fromXML(final XMLStreamReader sr) throws XMLStreamException { + loop: + while (sr.hasNext()) { + int eventType = sr.next(); + switch (eventType) { + case XMLStreamReader.START_ELEMENT: + String elementName = sr.getLocalName().toLowerCase(); + switch (elementName) { + case "enabled": + this.enabled = Boolean.parseBoolean(Cfg.readValue(sr)); + break; + case "current_count": + this.current = Integer.parseInt(Cfg.readValue(sr)); + // must be at least 128 + if (this.current < 128) { + this.current = 128; + } + break; + case "archive_rate": + this.archive = Integer.parseInt(Cfg.readValue(sr)); + // must be at least 1000 + if (this.current < 1000) { + this.current = 1000; + } + break; + default: + Cfg.skipElement(sr); + break; + } + break; + case XMLStreamReader.END_ELEMENT: + break loop; + } + } + } + + public void toXML(XMLStreamWriter xmlWriter) throws XMLStreamException { + xmlWriter.writeCharacters("\r\n\t\t"); + xmlWriter.writeStartElement("prune"); + + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeComment("Boolean value. Enable/disable database pruning."); + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeStartElement("enabled"); + xmlWriter.writeCharacters(String.valueOf(this.enabled)); + xmlWriter.writeEndElement(); + + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeComment( + "Integer value with minimum set to 128. Only blocks older than best block level minus this number are candidates for pruning."); + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeStartElement("current_count"); + xmlWriter.writeCharacters(String.valueOf(this.current)); + xmlWriter.writeEndElement(); + + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeComment( + "Integer value with minimum set to 1000. States for blocks that are exact multiples of this number will not be pruned."); + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeStartElement("archive_rate"); + xmlWriter.writeCharacters(String.valueOf(this.archive)); + xmlWriter.writeEndElement(); + + xmlWriter.writeCharacters("\r\n\t\t"); + xmlWriter.writeEndElement(); + } + + public boolean isEnabled() { + return enabled; + } + + @Override + public int getCurrentCount() { + return current; + } + + @Override + public int getArchiveRate() { + return archive; + } +} From e3198507212c39449e1e4a76113930fe3b03174d Mon Sep 17 00:00:00 2001 From: Alexandra Date: Tue, 15 May 2018 13:59:36 -0400 Subject: [PATCH 12/35] using new detailed pruning configuration --- .../org/aion/base/db/IRepositoryConfig.java | 12 +- .../aion/zero/impl/StandaloneBlockchain.java | 230 ++++++++++-------- .../aion/zero/impl/db/AionRepositoryImpl.java | 20 +- .../aion/zero/impl/db/RepositoryConfig.java | 24 +- .../org/aion/db/AionContractDetailsTest.java | 149 +++++++----- .../aion/zero/impl/MockRepositoryConfig.java | 9 +- .../zero/impl/db/AionRepositoryImplTest.java | 10 +- modBoot/resource/config.xml | 9 + modMcf/src/org/aion/mcf/config/CfgDb.java | 183 +++++++------- .../org/aion/mcf/db/AbstractRepository.java | 15 +- 10 files changed, 353 insertions(+), 308 deletions(-) diff --git a/modAionBase/src/org/aion/base/db/IRepositoryConfig.java b/modAionBase/src/org/aion/base/db/IRepositoryConfig.java index 02671bd267..7a315bbfd9 100644 --- a/modAionBase/src/org/aion/base/db/IRepositoryConfig.java +++ b/modAionBase/src/org/aion/base/db/IRepositoryConfig.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -37,19 +37,17 @@ import java.util.Properties; /** - * Represents a configuration interface accepted that should be accepted by the - * repository to implement necessary configs + * Represents a configuration interface accepted that should be accepted by the repository to + * implement necessary configs * * @author yao */ public interface IRepositoryConfig { - /** - * @return absolute path to the DB folder containing files - */ + /** @return absolute path to the DB folder containing files */ String getDbPath(); - int getPrune(); + IPruneConfig getPruneConfig(); IContractDetails contractDetailsImpl(); diff --git a/modAionImpl/src/org/aion/zero/impl/StandaloneBlockchain.java b/modAionImpl/src/org/aion/zero/impl/StandaloneBlockchain.java index 02608cb2e1..092a83d873 100644 --- a/modAionImpl/src/org/aion/zero/impl/StandaloneBlockchain.java +++ b/modAionImpl/src/org/aion/zero/impl/StandaloneBlockchain.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -19,22 +19,27 @@ * * Contributors: * Aion foundation. - * + * ******************************************************************************/ package org.aion.zero.impl; +import java.math.BigInteger; +import java.util.*; import org.aion.base.db.IContractDetails; +import org.aion.base.db.IPruneConfig; import org.aion.base.db.IRepositoryCache; import org.aion.base.db.IRepositoryConfig; import org.aion.base.type.Address; import org.aion.base.util.ByteArrayWrapper; -import org.aion.db.impl.DatabaseFactory; -import org.aion.mcf.core.AccountState; import org.aion.crypto.ECKey; import org.aion.crypto.ECKeyFac; import org.aion.db.impl.DBVendor; +import org.aion.db.impl.DatabaseFactory; +import org.aion.mcf.config.CfgPrune; +import org.aion.mcf.core.AccountState; import org.aion.mcf.valid.BlockHeaderValidator; +import org.aion.mcf.vm.types.DataWord; import org.aion.vm.PrecompiledContracts; import org.aion.zero.exceptions.HeaderStructureException; import org.aion.zero.impl.blockchain.ChainConfiguration; @@ -47,50 +52,49 @@ import org.aion.zero.impl.valid.AionHeaderVersionRule; import org.aion.zero.impl.valid.EnergyConsumedRule; import org.aion.zero.types.A0BlockHeader; -import org.aion.mcf.vm.types.DataWord; - -import java.math.BigInteger; -import java.util.*; /** - * Used mainly for debugging and testing purposes, provides codepaths for easy - * setup, into standard configurations that a user might expected, and handles - * any non-intuitive setup that the blockchain may require. + * Used mainly for debugging and testing purposes, provides codepaths for easy setup, into standard + * configurations that a user might expected, and handles any non-intuitive setup that the + * blockchain may require. */ public class StandaloneBlockchain extends AionBlockchainImpl { public AionGenesis genesis; - private static IRepositoryConfig repoConfig = new IRepositoryConfig() { - @Override - public String getDbPath() { - return ""; - } + private static IRepositoryConfig repoConfig = + new IRepositoryConfig() { + @Override + public String getDbPath() { + return ""; + } - @Override - public int getPrune() { - return -1; - } + @Override + public IPruneConfig getPruneConfig() { + return new CfgPrune(false); + } - @Override - public IContractDetails contractDetailsImpl() { - return ContractDetailsAion.createForTesting(0, 1000000).getDetails(); - } + @Override + public IContractDetails contractDetailsImpl() { + return ContractDetailsAion.createForTesting(0, 1000000).getDetails(); + } - @Override - public Properties getDatabaseConfig(String db_name) { - Properties props = new Properties(); - props.setProperty(DatabaseFactory.Props.DB_TYPE, DBVendor.MOCKDB.toValue()); - props.setProperty(DatabaseFactory.Props.ENABLE_HEAP_CACHE, "false"); - return props; - } - }; + @Override + public Properties getDatabaseConfig(String db_name) { + Properties props = new Properties(); + props.setProperty(DatabaseFactory.Props.DB_TYPE, DBVendor.MOCKDB.toValue()); + props.setProperty(DatabaseFactory.Props.ENABLE_HEAP_CACHE, "false"); + return props; + } + }; protected StandaloneBlockchain(final A0BCConfig config, final ChainConfiguration chainConfig) { super(config, AionRepositoryImpl.createForTesting(repoConfig), chainConfig); } - protected StandaloneBlockchain(final A0BCConfig config, final ChainConfiguration chainConfig, + protected StandaloneBlockchain( + final A0BCConfig config, + final ChainConfiguration chainConfig, IRepositoryConfig repoConfig) { super(config, AionRepositoryImpl.createForTesting(repoConfig), chainConfig); } @@ -103,9 +107,7 @@ public AionGenesis getGenesis() { return this.genesis; } - public void loadJSON(String json) { - - } + public void loadJSON(String json) {} public BlockHeaderValidator getBlockHeaderValidator() { return this.chainConfiguration.createBlockHeaderValidator(); @@ -133,15 +135,15 @@ public static class Builder { private IRepositoryConfig repoConfig; public static final int INITIAL_ACC_LEN = 10; - public static final BigInteger DEFAULT_BALANCE = new BigInteger("1000000000000000000000000"); + public static final BigInteger DEFAULT_BALANCE = + new BigInteger("1000000000000000000000000"); /** - * The type of validator selected for the blockchain, a "full" validator - * validates blocks as if they were broadcasted from the network. - * Therefore the header validator will require a valid equiHash - * solution. + * The type of validator selected for the blockchain, a "full" validator validates blocks as + * if they were broadcasted from the network. Therefore the header validator will require a + * valid equiHash solution. * - * {@code validatorType -> (full|simple)} + *

{@code validatorType -> (full|simple)} */ String validatorType; @@ -154,7 +156,8 @@ public Builder withDefaultAccounts() { for (int i = 0; i < INITIAL_ACC_LEN; i++) { ECKey pk = ECKeyFac.inst().create(); this.defaultKeys.add(pk); - initialState.put(new ByteArrayWrapper(pk.getAddress()), + initialState.put( + new ByteArrayWrapper(pk.getAddress()), new AccountState(BigInteger.ZERO, DEFAULT_BALANCE)); } return this; @@ -162,8 +165,11 @@ public Builder withDefaultAccounts() { public Builder withDefaultAccounts(List defaultAccounts) { this.defaultKeys.addAll(defaultAccounts); - this.defaultKeys.forEach(k -> initialState.put(new ByteArrayWrapper(k.getAddress()), - new AccountState(BigInteger.ZERO, DEFAULT_BALANCE))); + this.defaultKeys.forEach( + k -> + initialState.put( + new ByteArrayWrapper(k.getAddress()), + new AccountState(BigInteger.ZERO, DEFAULT_BALANCE))); return this; } @@ -199,8 +205,8 @@ public String getDbPath() { } @Override - public int getPrune() { - return -1; + public IPruneConfig getPruneConfig() { + return new CfgPrune(false); } @Override @@ -219,41 +225,45 @@ public Properties getDatabaseConfig(String db_name) { } public Bundle build() { - this.a0Config = this.a0Config == null ? new A0BCConfig() { - @Override - public Address getCoinbase() { - return Address.ZERO_ADDRESS(); - } - - @Override - public byte[] getExtraData() { - return new byte[32]; - } - - @Override - public boolean getExitOnBlockConflict() { - return false; - } - - @Override - public Address getMinerCoinbase() { - return Address.ZERO_ADDRESS(); - } - - @Override - public int getFlushInterval() { - return 1; - } - - @Override - public AbstractEnergyStrategyLimit getEnergyLimitStrategy() { - return new TargetStrategy( - configuration.getConstants().getEnergyLowerBoundLong(), - configuration.getConstants().getEnergyDivisorLimitLong(), - 10_000_000L); - } - - } : this.a0Config; + this.a0Config = + this.a0Config == null + ? new A0BCConfig() { + @Override + public Address getCoinbase() { + return Address.ZERO_ADDRESS(); + } + + @Override + public byte[] getExtraData() { + return new byte[32]; + } + + @Override + public boolean getExitOnBlockConflict() { + return false; + } + + @Override + public Address getMinerCoinbase() { + return Address.ZERO_ADDRESS(); + } + + @Override + public int getFlushInterval() { + return 1; + } + + @Override + public AbstractEnergyStrategyLimit getEnergyLimitStrategy() { + return new TargetStrategy( + configuration.getConstants().getEnergyLowerBoundLong(), + configuration + .getConstants() + .getEnergyDivisorLimitLong(), + 10_000_000L); + } + } + : this.a0Config; if (this.configuration == null) { if (this.validatorType == null) { @@ -261,34 +271,38 @@ public AbstractEnergyStrategyLimit getEnergyLimitStrategy() { } else if (this.validatorType.equals("full")) { this.configuration = new ChainConfiguration(); } else if (this.validatorType.equals("simple")) { - this.configuration = new ChainConfiguration() { - /* - * Remove the equiHash solution for the simplified - * validator this gives us the ability to connect new - * blocks without validating the solution and POW. - * - * This is good for transaction testing, but another set - * of tests need to ensure that the equihash and POW - * generated are valid. - */ - @Override - public BlockHeaderValidator createBlockHeaderValidator() { - return new BlockHeaderValidator( - Arrays.asList( - new AionExtraDataRule(this.constants.getMaximumExtraDataSize()), - new EnergyConsumedRule(), - new AionHeaderVersionRule())); - } - }; + this.configuration = + new ChainConfiguration() { + /* + * Remove the equiHash solution for the simplified + * validator this gives us the ability to connect new + * blocks without validating the solution and POW. + * + * This is good for transaction testing, but another set + * of tests need to ensure that the equihash and POW + * generated are valid. + */ + @Override + public BlockHeaderValidator + createBlockHeaderValidator() { + return new BlockHeaderValidator( + Arrays.asList( + new AionExtraDataRule( + this.constants + .getMaximumExtraDataSize()), + new EnergyConsumedRule(), + new AionHeaderVersionRule())); + } + }; } else { throw new IllegalArgumentException("validatorType != (full|simple)"); } } - if (this.repoConfig == null) - this.repoConfig = generateRepositoryConfig(); + if (this.repoConfig == null) this.repoConfig = generateRepositoryConfig(); - StandaloneBlockchain bc = new StandaloneBlockchain(this.a0Config, this.configuration, this.repoConfig); + StandaloneBlockchain bc = + new StandaloneBlockchain(this.a0Config, this.configuration, this.repoConfig); AionGenesis.Builder genesisBuilder = new AionGenesis.Builder(); for (Map.Entry acc : this.initialState.entrySet()) { @@ -307,7 +321,9 @@ public BlockHeaderValidator createBlockHeaderValidator() { track.createAccount(PrecompiledContracts.totalCurrencyAddress); for (Map.Entry key : genesis.getNetworkBalances().entrySet()) { - track.addStorageRow(PrecompiledContracts.totalCurrencyAddress, new DataWord(key.getKey()), + track.addStorageRow( + PrecompiledContracts.totalCurrencyAddress, + new DataWord(key.getKey()), new DataWord(key.getValue())); } @@ -320,8 +336,8 @@ public BlockHeaderValidator createBlockHeaderValidator() { // TODO: violates abstraction, consider adding to interface after // stable ((AionRepositoryImpl) bc.getRepository()).commitBlock(genesis.getHeader()); - ((AionBlockStore) bc.getRepository().getBlockStore()).saveBlock(genesis, genesis.getCumulativeDifficulty(), - true); + ((AionBlockStore) bc.getRepository().getBlockStore()) + .saveBlock(genesis, genesis.getCumulativeDifficulty(), true); bc.setBestBlock(genesis); bc.setTotalDifficulty(genesis.getCumulativeDifficulty()); diff --git a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java index 7b2f58e103..f1195566bb 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java +++ b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java @@ -74,17 +74,6 @@ private static class AionRepositoryImplHolder { new RepositoryConfig( new File(config.getBasePath(), config.getDb().getPath()) .getAbsolutePath(), - config.getDb().getPrune() > 0 - ? - // if the value is smaller than backward step - // there is the risk of importing state-less blocks after - // reboot - (128 > config.getDb().getPrune() - ? 128 - : config.getDb().getPrune()) - : - // negative value => pruning disabled - config.getDb().getPrune(), ContractDetailsAion.getInstance(), config.getDb())); } @@ -122,7 +111,7 @@ public TransactionStore getTransacti } private Trie createStateTrie() { - return new SecureTrie(stateDSPrune).withPruningEnabled(pruneBlockCount > 0); + return new SecureTrie(stateDSPrune).withPruningEnabled(pruneEnabled); } @Override @@ -512,7 +501,7 @@ public void commitBlock(A0BlockHeader blockHeader) { worldState.sync(); detailsDS.syncLargeStorage(); - if (pruneBlockCount > 0) { + if (pruneEnabled) { stateDSPrune.storeBlockChanges(blockHeader.getHash(), blockHeader.getNumber()); detailsDS .getStorageDSPrune() @@ -554,7 +543,12 @@ public IRepository getSnapshotTo(byte[] root) { repo.cfg = cfg; repo.stateDatabase = this.stateDatabase; repo.stateDSPrune = this.stateDSPrune; + + // pruning config + repo.pruneEnabled = this.pruneEnabled; repo.pruneBlockCount = this.pruneBlockCount; + repo.archiveRate = this.archiveRate; + repo.detailsDS = this.detailsDS; repo.isSnapshot = true; diff --git a/modAionImpl/src/org/aion/zero/impl/db/RepositoryConfig.java b/modAionImpl/src/org/aion/zero/impl/db/RepositoryConfig.java index a2acc69722..c65456edcb 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/RepositoryConfig.java +++ b/modAionImpl/src/org/aion/zero/impl/db/RepositoryConfig.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -19,23 +19,21 @@ * * Contributors: * Aion foundation. - * ******************************************************************************/ - package org.aion.zero.impl.db; +import java.util.Map; +import java.util.Properties; import org.aion.base.db.DetailsProvider; import org.aion.base.db.IContractDetails; +import org.aion.base.db.IPruneConfig; import org.aion.base.db.IRepositoryConfig; import org.aion.mcf.config.CfgDb; -import java.util.Map; -import java.util.Properties; - public class RepositoryConfig implements IRepositoryConfig { private final String dbPath; - private final int prune; + private final IPruneConfig cfgPrune; private final DetailsProvider detailsProvider; private final Map cfg; @@ -45,8 +43,8 @@ public String getDbPath() { } @Override - public int getPrune() { - return prune; + public IPruneConfig getPruneConfig() { + return cfgPrune; } @Override @@ -63,13 +61,11 @@ public Properties getDatabaseConfig(String db_name) { return new Properties(prop); } - public RepositoryConfig(final String dbPath, - final int prune, - final DetailsProvider detailsProvider, - final CfgDb cfgDb) { + public RepositoryConfig( + final String dbPath, final DetailsProvider detailsProvider, final CfgDb cfgDb) { this.dbPath = dbPath; - this.prune = prune; this.detailsProvider = detailsProvider; this.cfg = cfgDb.asProperties(); + this.cfgPrune = cfgDb.getPrune(); } } diff --git a/modAionImpl/test/org/aion/db/AionContractDetailsTest.java b/modAionImpl/test/org/aion/db/AionContractDetailsTest.java index 4222d694fc..5c69f27978 100644 --- a/modAionImpl/test/org/aion/db/AionContractDetailsTest.java +++ b/modAionImpl/test/org/aion/db/AionContractDetailsTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -34,59 +34,61 @@ ******************************************************************************/ package org.aion.db; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.base.db.IContractDetails; +import org.aion.base.db.IPruneConfig; import org.aion.base.db.IRepositoryConfig; import org.aion.base.type.Address; import org.aion.base.util.ByteUtil; import org.aion.db.impl.DBVendor; import org.aion.db.impl.DatabaseFactory; -import org.aion.db.impl.leveldb.LevelDBConstants; +import org.aion.mcf.config.CfgPrune; import org.aion.mcf.vm.types.DataWord; import org.aion.zero.db.AionContractDetailsImpl; import org.aion.zero.impl.db.AionRepositoryImpl; import org.aion.zero.impl.db.ContractDetailsAion; import org.apache.commons.lang3.RandomUtils; -import org.junit.Ignore; import org.junit.Test; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - public class AionContractDetailsTest { - private static final int IN_MEMORY_STORAGE_LIMIT = 1000000; // CfgAion.inst().getDb().getDetailsInMemoryStorageLimit(); - - protected IRepositoryConfig repoConfig = new IRepositoryConfig() { - @Override - public String getDbPath() { - return ""; - } - - @Override - public int getPrune() { - return 0; - } - - @Override - public IContractDetails contractDetailsImpl() { - return ContractDetailsAion.createForTesting(0, 1000000).getDetails(); - } - - @Override - public Properties getDatabaseConfig(String db_name) { - Properties props = new Properties(); - props.setProperty(DatabaseFactory.Props.DB_TYPE, DBVendor.MOCKDB.toValue()); - props.setProperty(DatabaseFactory.Props.ENABLE_HEAP_CACHE, "false"); - return props; - } - }; - - private static IContractDetails deserialize(byte[] rlp, IByteArrayKeyValueDatabase externalStorage) { + private static final int IN_MEMORY_STORAGE_LIMIT = + 1000000; // CfgAion.inst().getDb().getDetailsInMemoryStorageLimit(); + + protected IRepositoryConfig repoConfig = + new IRepositoryConfig() { + @Override + public String getDbPath() { + return ""; + } + + @Override + public IPruneConfig getPruneConfig() { + return new CfgPrune(false); + } + + @Override + public IContractDetails contractDetailsImpl() { + return ContractDetailsAion.createForTesting(0, 1000000).getDetails(); + } + + @Override + public Properties getDatabaseConfig(String db_name) { + Properties props = new Properties(); + props.setProperty(DatabaseFactory.Props.DB_TYPE, DBVendor.MOCKDB.toValue()); + props.setProperty(DatabaseFactory.Props.ENABLE_HEAP_CACHE, "false"); + return props; + } + }; + + private static IContractDetails deserialize( + byte[] rlp, IByteArrayKeyValueDatabase externalStorage) { AionContractDetailsImpl result = new AionContractDetailsImpl(); result.setExternalStorageDataSource(externalStorage); result.decode(rlp); @@ -105,10 +107,11 @@ public void test_1() throws Exception { byte[] key_2 = ByteUtil.hexStringToBytes("222222"); byte[] val_2 = ByteUtil.hexStringToBytes("bbbbbb"); - AionContractDetailsImpl contractDetails = new AionContractDetailsImpl( - -1, //CfgAion.inst().getDb().getPrune(), - 1000000 //CfgAion.inst().getDb().getDetailsInMemoryStorageLimit() - ); + AionContractDetailsImpl contractDetails = + new AionContractDetailsImpl( + -1, // CfgAion.inst().getDb().getPrune(), + 1000000 // CfgAion.inst().getDb().getDetailsInMemoryStorageLimit() + ); contractDetails.setCode(code); contractDetails.put(new DataWord(key_1), new DataWord(val_1)); contractDetails.put(new DataWord(key_2), new DataWord(val_2)); @@ -117,20 +120,25 @@ public void test_1() throws Exception { AionContractDetailsImpl contractDetails_ = new AionContractDetailsImpl(data); - assertEquals(ByteUtil.toHexString(code), - ByteUtil.toHexString(contractDetails_.getCode())); + assertEquals(ByteUtil.toHexString(code), ByteUtil.toHexString(contractDetails_.getCode())); - assertEquals(ByteUtil.toHexString(val_1), - ByteUtil.toHexString(contractDetails_.get(new DataWord(key_1)).getNoLeadZeroesData())); + assertEquals( + ByteUtil.toHexString(val_1), + ByteUtil.toHexString( + contractDetails_.get(new DataWord(key_1)).getNoLeadZeroesData())); - assertEquals(ByteUtil.toHexString(val_2), - ByteUtil.toHexString(contractDetails_.get(new DataWord(key_2)).getNoLeadZeroesData())); + assertEquals( + ByteUtil.toHexString(val_2), + ByteUtil.toHexString( + contractDetails_.get(new DataWord(key_2)).getNoLeadZeroesData())); } @Test public void test_2() throws Exception { - byte[] code = ByteUtil.hexStringToBytes("7c0100000000000000000000000000000000000000000000000000000000600035046333d546748114610065578063430fe5f01461007c5780634d432c1d1461008d578063501385b2146100b857806357eb3b30146100e9578063dbc7df61146100fb57005b6100766004356024356044356102f0565b60006000f35b61008760043561039e565b60006000f35b610098600435610178565b8073ffffffffffffffffffffffffffffffffffffffff1660005260206000f35b6100c96004356024356044356101a0565b8073ffffffffffffffffffffffffffffffffffffffff1660005260206000f35b6100f1610171565b8060005260206000f35b610106600435610133565b8360005282602052816040528073ffffffffffffffffffffffffffffffffffffffff1660605260806000f35b5b60006020819052908152604090208054600182015460028301546003909301549192909173ffffffffffffffffffffffffffffffffffffffff1684565b5b60015481565b5b60026020526000908152604090205473ffffffffffffffffffffffffffffffffffffffff1681565b73ffffffffffffffffffffffffffffffffffffffff831660009081526020819052604081206002015481908302341080156101fe575073ffffffffffffffffffffffffffffffffffffffff8516600090815260208190526040812054145b8015610232575073ffffffffffffffffffffffffffffffffffffffff85166000908152602081905260409020600101548390105b61023b57610243565b3391506102e8565b6101966103ca60003973ffffffffffffffffffffffffffffffffffffffff3381166101965285166101b68190526000908152602081905260408120600201546101d6526101f68490526102169080f073ffffffffffffffffffffffffffffffffffffffff8616600090815260208190526040902060030180547fffffffffffffffffffffffff0000000000000000000000000000000000000000168217905591508190505b509392505050565b73ffffffffffffffffffffffffffffffffffffffff33166000908152602081905260408120548190821461032357610364565b60018054808201909155600090815260026020526040902080547fffffffffffffffffffffffff000000000000000000000000000000000000000016331790555b50503373ffffffffffffffffffffffffffffffffffffffff1660009081526020819052604090209081556001810192909255600290910155565b3373ffffffffffffffffffffffffffffffffffffffff166000908152602081905260409020600201555600608061019660043960048051602451604451606451600080547fffffffffffffffffffffffff0000000000000000000000000000000000000000908116909517815560018054909516909317909355600355915561013390819061006390396000f3007c0100000000000000000000000000000000000000000000000000000000600035046347810fe381146100445780637e4a1aa81461005557806383d2421b1461006957005b61004f6004356100ab565b60006000f35b6100636004356024356100fc565b60006000f35b61007460043561007a565b60006000f35b6001543373ffffffffffffffffffffffffffffffffffffffff9081169116146100a2576100a8565b60078190555b50565b73ffffffffffffffffffffffffffffffffffffffff8116600090815260026020526040902080547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0016600117905550565b6001543373ffffffffffffffffffffffffffffffffffffffff9081169116146101245761012f565b600582905560068190555b505056"); + byte[] code = + ByteUtil.hexStringToBytes( + "7c0100000000000000000000000000000000000000000000000000000000600035046333d546748114610065578063430fe5f01461007c5780634d432c1d1461008d578063501385b2146100b857806357eb3b30146100e9578063dbc7df61146100fb57005b6100766004356024356044356102f0565b60006000f35b61008760043561039e565b60006000f35b610098600435610178565b8073ffffffffffffffffffffffffffffffffffffffff1660005260206000f35b6100c96004356024356044356101a0565b8073ffffffffffffffffffffffffffffffffffffffff1660005260206000f35b6100f1610171565b8060005260206000f35b610106600435610133565b8360005282602052816040528073ffffffffffffffffffffffffffffffffffffffff1660605260806000f35b5b60006020819052908152604090208054600182015460028301546003909301549192909173ffffffffffffffffffffffffffffffffffffffff1684565b5b60015481565b5b60026020526000908152604090205473ffffffffffffffffffffffffffffffffffffffff1681565b73ffffffffffffffffffffffffffffffffffffffff831660009081526020819052604081206002015481908302341080156101fe575073ffffffffffffffffffffffffffffffffffffffff8516600090815260208190526040812054145b8015610232575073ffffffffffffffffffffffffffffffffffffffff85166000908152602081905260409020600101548390105b61023b57610243565b3391506102e8565b6101966103ca60003973ffffffffffffffffffffffffffffffffffffffff3381166101965285166101b68190526000908152602081905260408120600201546101d6526101f68490526102169080f073ffffffffffffffffffffffffffffffffffffffff8616600090815260208190526040902060030180547fffffffffffffffffffffffff0000000000000000000000000000000000000000168217905591508190505b509392505050565b73ffffffffffffffffffffffffffffffffffffffff33166000908152602081905260408120548190821461032357610364565b60018054808201909155600090815260026020526040902080547fffffffffffffffffffffffff000000000000000000000000000000000000000016331790555b50503373ffffffffffffffffffffffffffffffffffffffff1660009081526020819052604090209081556001810192909255600290910155565b3373ffffffffffffffffffffffffffffffffffffffff166000908152602081905260409020600201555600608061019660043960048051602451604451606451600080547fffffffffffffffffffffffff0000000000000000000000000000000000000000908116909517815560018054909516909317909355600355915561013390819061006390396000f3007c0100000000000000000000000000000000000000000000000000000000600035046347810fe381146100445780637e4a1aa81461005557806383d2421b1461006957005b61004f6004356100ab565b60006000f35b6100636004356024356100fc565b60006000f35b61007460043561007a565b60006000f35b6001543373ffffffffffffffffffffffffffffffffffffffff9081169116146100a2576100a8565b60078190555b50565b73ffffffffffffffffffffffffffffffffffffffff8116600090815260026020526040902080547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0016600117905550565b6001543373ffffffffffffffffffffffffffffffffffffffff9081169116146101245761012f565b600582905560068190555b505056"); Address address = Address.wrap(RandomUtils.nextBytes(Address.ADDRESS_LEN)); byte[] key_0 = ByteUtil.hexStringToBytes("18d63b70aa690ad37cb50908746c9a55"); @@ -197,48 +205,60 @@ public void test_2() throws Exception { AionContractDetailsImpl contractDetails_ = new AionContractDetailsImpl(data); - assertEquals(ByteUtil.toHexString(code), - ByteUtil.toHexString(contractDetails_.getCode())); + assertEquals(ByteUtil.toHexString(code), ByteUtil.toHexString(contractDetails_.getCode())); assertTrue(address.equals(contractDetails_.getAddress())); - assertEquals(ByteUtil.toHexString(val_1), + assertEquals( + ByteUtil.toHexString(val_1), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_1)).getData())); - assertEquals(ByteUtil.toHexString(val_2), + assertEquals( + ByteUtil.toHexString(val_2), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_2)).getData())); - assertEquals(ByteUtil.toHexString(val_3), + assertEquals( + ByteUtil.toHexString(val_3), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_3)).getData())); - assertEquals(ByteUtil.toHexString(val_4), + assertEquals( + ByteUtil.toHexString(val_4), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_4)).getData())); - assertEquals(ByteUtil.toHexString(val_5), + assertEquals( + ByteUtil.toHexString(val_5), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_5)).getData())); - assertEquals(ByteUtil.toHexString(val_6), + assertEquals( + ByteUtil.toHexString(val_6), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_6)).getData())); - assertEquals(ByteUtil.toHexString(val_7), + assertEquals( + ByteUtil.toHexString(val_7), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_7)).getData())); - assertEquals(ByteUtil.toHexString(val_8), + assertEquals( + ByteUtil.toHexString(val_8), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_8)).getData())); - assertEquals(ByteUtil.toHexString(val_9), + assertEquals( + ByteUtil.toHexString(val_9), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_9)).getData())); - assertEquals(ByteUtil.toHexString(val_10), + assertEquals( + ByteUtil.toHexString(val_10), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_10)).getData())); - assertEquals(ByteUtil.toHexString(val_11), + assertEquals( + ByteUtil.toHexString(val_11), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_11)).getData())); - assertEquals(ByteUtil.toHexString(val_12), + assertEquals( + ByteUtil.toHexString(val_12), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_12)).getData())); - assertEquals(ByteUtil.toHexString(val_13), + assertEquals( + ByteUtil.toHexString(val_13), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_13)).getData())); } @@ -310,7 +330,6 @@ public void testExternalStorageTransition() { original.put(key, value); } - original.syncStorage(); assertTrue(!externalStorage.isEmpty()); diff --git a/modAionImpl/test/org/aion/zero/impl/MockRepositoryConfig.java b/modAionImpl/test/org/aion/zero/impl/MockRepositoryConfig.java index 699260162a..15a29ea253 100644 --- a/modAionImpl/test/org/aion/zero/impl/MockRepositoryConfig.java +++ b/modAionImpl/test/org/aion/zero/impl/MockRepositoryConfig.java @@ -1,13 +1,14 @@ package org.aion.zero.impl; +import java.util.Properties; import org.aion.base.db.IContractDetails; +import org.aion.base.db.IPruneConfig; import org.aion.base.db.IRepositoryConfig; import org.aion.db.impl.DBVendor; import org.aion.db.impl.DatabaseFactory; +import org.aion.mcf.config.CfgPrune; import org.aion.zero.impl.db.ContractDetailsAion; -import java.util.Properties; - public class MockRepositoryConfig implements IRepositoryConfig { private DBVendor vendor = DBVendor.MOCKDB; @@ -17,8 +18,8 @@ public String getDbPath() { } @Override - public int getPrune() { - return 0; + public IPruneConfig getPruneConfig() { + return new CfgPrune(false); } @Override diff --git a/modAionImpl/test/org/aion/zero/impl/db/AionRepositoryImplTest.java b/modAionImpl/test/org/aion/zero/impl/db/AionRepositoryImplTest.java index 412e60b5b6..c05c59b414 100644 --- a/modAionImpl/test/org/aion/zero/impl/db/AionRepositoryImplTest.java +++ b/modAionImpl/test/org/aion/zero/impl/db/AionRepositoryImplTest.java @@ -39,15 +39,13 @@ import java.math.BigInteger; import java.util.Optional; import java.util.Properties; -import org.aion.base.db.IByteArrayKeyValueDatabase; -import org.aion.base.db.IContractDetails; -import org.aion.base.db.IRepositoryCache; -import org.aion.base.db.IRepositoryConfig; +import org.aion.base.db.*; import org.aion.base.type.Address; import org.aion.base.util.ByteUtil; import org.aion.crypto.HashUtil; import org.aion.db.impl.DBVendor; import org.aion.db.impl.DatabaseFactory; +import org.aion.mcf.config.CfgPrune; import org.aion.mcf.core.AccountState; import org.aion.mcf.db.IBlockStoreBase; import org.aion.mcf.vm.types.DataWord; @@ -67,8 +65,8 @@ public String getDbPath() { } @Override - public int getPrune() { - return 0; + public IPruneConfig getPruneConfig() { + return new CfgPrune(false); } @Override diff --git a/modBoot/resource/config.xml b/modBoot/resource/config.xml index 9b3a51e4e9..a3234cb7a4 100644 --- a/modBoot/resource/config.xml +++ b/modBoot/resource/config.xml @@ -64,6 +64,15 @@ database true + + + + true + + 128 + + 10000 + 1000 diff --git a/modMcf/src/org/aion/mcf/config/CfgDb.java b/modMcf/src/org/aion/mcf/config/CfgDb.java index 9f39813acd..b1272ddccc 100644 --- a/modMcf/src/org/aion/mcf/config/CfgDb.java +++ b/modMcf/src/org/aion/mcf/config/CfgDb.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -22,25 +22,22 @@ ******************************************************************************/ package org.aion.mcf.config; -import org.aion.base.util.Utils; -import org.aion.db.impl.DBVendor; +import static org.aion.db.impl.DatabaseFactory.Props; -import javax.xml.stream.XMLOutputFactory; -import javax.xml.stream.XMLStreamException; -import javax.xml.stream.XMLStreamReader; -import javax.xml.stream.XMLStreamWriter; import java.io.IOException; import java.io.StringWriter; import java.io.Writer; import java.util.HashMap; import java.util.Map; import java.util.Properties; +import javax.xml.stream.XMLOutputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import javax.xml.stream.XMLStreamWriter; +import org.aion.base.util.Utils; +import org.aion.db.impl.DBVendor; -import static org.aion.db.impl.DatabaseFactory.Props; - -/** - * @author chris - */ +/** @author chris */ public class CfgDb { public static class Names { @@ -63,7 +60,7 @@ public static class Names { private String vendor; private boolean compression; private boolean check_integrity; - private int prune; + private CfgPrune prune; /** * Enabling expert mode allows more detailed database configurations. @@ -80,7 +77,7 @@ public CfgDb() { this.vendor = DBVendor.LEVELDB.toValue(); this.compression = false; this.check_integrity = true; - this.prune = -1; + this.prune = new CfgPrune(); if (expert) { this.specificConfig = new HashMap<>(); @@ -103,79 +100,88 @@ public void fromXML(final XMLStreamReader sr) throws XMLStreamException { this.check_integrity = Boolean.parseBoolean(Cfg.readValue(sr)); break; case "prune": - this.prune = Integer.parseInt(Cfg.readValue(sr)); + this.prune.fromXML(sr); break; - // parameter considered only when expert==false + // parameter considered only when expert==false case "vendor": this.vendor = Cfg.readValue(sr); break; - // parameter considered only when expert==false + // parameter considered only when expert==false case Props.ENABLE_DB_COMPRESSION: this.compression = Boolean.parseBoolean(Cfg.readValue(sr)); break; - // parameter considered only when expert==true - case Names.DEFAULT: { - CfgDbDetails dbConfig = this.specificConfig.get(Names.DEFAULT); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.DEFAULT, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.BLOCK: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.BLOCK, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.INDEX: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.INDEX, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.DETAILS: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.DETAILS, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.STORAGE: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.STORAGE, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.STATE: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.STATE, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.TRANSACTION: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.TRANSACTION, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.TX_POOL: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.TX_POOL, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.TX_CACHE: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.TX_CACHE, dbConfig); - break; - } + // parameter considered only when expert==true + case Names.DEFAULT: + { + CfgDbDetails dbConfig = this.specificConfig.get(Names.DEFAULT); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.DEFAULT, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.BLOCK: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.BLOCK, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.INDEX: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.INDEX, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.DETAILS: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.DETAILS, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.STORAGE: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.STORAGE, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.STATE: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.STATE, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.TRANSACTION: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.TRANSACTION, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.TX_POOL: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.TX_POOL, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.TX_CACHE: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.TX_CACHE, dbConfig); + break; + } default: Cfg.skipElement(sr); break; @@ -205,25 +211,24 @@ public String toXML() { xmlWriter.writeEndElement(); xmlWriter.writeCharacters("\r\n\t\t"); - xmlWriter.writeComment("Boolean value. Enable/disable database integrity check run at startup."); + xmlWriter.writeComment( + "Boolean value. Enable/disable database integrity check run at startup."); xmlWriter.writeCharacters("\r\n\t\t"); xmlWriter.writeStartElement("check_integrity"); xmlWriter.writeCharacters(String.valueOf(this.check_integrity)); xmlWriter.writeEndElement(); xmlWriter.writeCharacters("\r\n\t\t"); - xmlWriter.writeComment("Integer value. Number of blocks after which to prune. Pruning disabled when negative."); - xmlWriter.writeCharacters("\r\n\t\t"); - xmlWriter.writeStartElement("prune"); - xmlWriter.writeCharacters(String.valueOf(this.prune)); - xmlWriter.writeEndElement(); + xmlWriter.writeComment("Configuration for data pruning behavior."); + this.prune.toXML(xmlWriter); if (!expert) { xmlWriter.writeCharacters("\r\n\t\t"); xmlWriter.writeComment( "Database implementation used to store data; supported options: leveldb, h2, rocksdb."); xmlWriter.writeCharacters("\r\n\t\t"); - xmlWriter.writeComment("Caution: changing implementation requires re-syncing from genesis!"); + xmlWriter.writeComment( + "Caution: changing implementation requires re-syncing from genesis!"); xmlWriter.writeCharacters("\r\n\t\t"); xmlWriter.writeStartElement("vendor"); xmlWriter.writeCharacters(this.vendor); @@ -260,7 +265,7 @@ public String getPath() { return this.path; } - public int getPrune() { + public CfgPrune getPrune() { return this.prune; } @@ -307,4 +312,4 @@ public void setHeapCacheEnabled(boolean value) { } } } -} \ No newline at end of file +} diff --git a/modMcf/src/org/aion/mcf/db/AbstractRepository.java b/modMcf/src/org/aion/mcf/db/AbstractRepository.java index 45bc3429a1..15246e6c98 100644 --- a/modMcf/src/org/aion/mcf/db/AbstractRepository.java +++ b/modMcf/src/org/aion/mcf/db/AbstractRepository.java @@ -102,6 +102,7 @@ public abstract class AbstractRepository< // Block related parameters. protected long bestBlockNumber = 0; protected long pruneBlockCount; + protected long archiveRate; protected boolean pruneEnabled = true; // Current blockstore. @@ -248,9 +249,17 @@ protected void initializeDatabasesAndCaches() throws Exception { // Setup the cache for transaction data source. this.detailsDS = new DetailsDataStore<>(detailsDatabase, storageDatabase, this.cfg); stateDSPrune = new JournalPruneDataSource(stateDatabase); - pruneBlockCount = pruneEnabled ? this.cfg.getPrune() : -1; - if (pruneEnabled && pruneBlockCount > 0) { - LOGGEN.info("Pruning block count set to {}.", pruneBlockCount); + + // pruning config + pruneEnabled = this.cfg.getPruneConfig().isEnabled(); + pruneBlockCount = this.cfg.getPruneConfig().getCurrentCount(); + archiveRate = this.cfg.getPruneConfig().getArchiveRate(); + + if (pruneEnabled) { + LOGGEN.info( + "Pruning ENABLED. Block count set to {} and archive rate set to {}.", + pruneBlockCount, + archiveRate); } else { stateDSPrune.setPruneEnabled(false); } From cf6e9d589a9a01b6f91dac2639f2eee971fc9177 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Tue, 15 May 2018 16:12:50 -0400 Subject: [PATCH 13/35] bugfix: method parameters should be generic --- .../src/org/aion/base/db/IKeyValueStore.java | 118 +++++++----------- 1 file changed, 47 insertions(+), 71 deletions(-) diff --git a/modAionBase/src/org/aion/base/db/IKeyValueStore.java b/modAionBase/src/org/aion/base/db/IKeyValueStore.java index 5f2f8c00ba..831d8257a6 100644 --- a/modAionBase/src/org/aion/base/db/IKeyValueStore.java +++ b/modAionBase/src/org/aion/base/db/IKeyValueStore.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -42,14 +42,11 @@ /** * Functionality for a key-value cache allowing itemized updates. * - * @param - * the data type of the keys - * @param - * the data type of the values + * @param the data type of the keys + * @param the data type of the values * @author Alexandra Roatis - * @implNote For the underlying DB connection, if [isClosed() == true], then all - * function calls which are documented to throw RuntimeException, will - * throw a RuntimeException. + * @implNote For the underlying DB connection, if [isClosed() == true], then all function calls + * which are documented to throw RuntimeException, will throw a RuntimeException. */ public interface IKeyValueStore extends AutoCloseable { @@ -60,8 +57,7 @@ public interface IKeyValueStore extends AutoCloseable { * Returns if the DB is empty or not. * * @return True if number of keys > 0, false otherwise - * @throws RuntimeException - * if the data store is closed + * @throws RuntimeException if the data store is closed */ boolean isEmpty(); @@ -69,23 +65,18 @@ public interface IKeyValueStore extends AutoCloseable { * Returns the set of keys for the database. * * @return Set of keys - * @throws RuntimeException - * if the data store is closed - * @apiNote Returns an empty set if the database keys could not be - * retrieved. + * @throws RuntimeException if the data store is closed + * @apiNote Returns an empty set if the database keys could not be retrieved. */ Set keys(); /** - * get retrieves a value from the database, returning an optional, it is - * fulfilled if a value was able to be retrieved from the DB, otherwise the - * optional is empty + * get retrieves a value from the database, returning an optional, it is fulfilled if a value + * was able to be retrieved from the DB, otherwise the optional is empty * * @param k - * @throws RuntimeException - * if the data store is closed - * @throws IllegalArgumentException - * if the key is null + * @throws RuntimeException if the data store is closed + * @throws IllegalArgumentException if the key is null */ Optional get(K k); @@ -93,44 +84,34 @@ public interface IKeyValueStore extends AutoCloseable { // ------------------------------------------------------------------------------------- /** - * Places or updates a value into the cache at the corresponding key. Makes - * no guarantees about when the value is actually inserted into the - * underlying data store. + * Places or updates a value into the cache at the corresponding key. Makes no guarantees about + * when the value is actually inserted into the underlying data store. * - * @param k - * the key for the new entry - * @param v - * the value for the new entry - * @throws RuntimeException - * if the underlying data store is closed - * @throws IllegalArgumentException - * if the key is null - * @implNote The choice of when to push the changes to the data store is - * left up to the implementation. + * @param k the key for the new entry + * @param v the value for the new entry + * @throws RuntimeException if the underlying data store is closed + * @throws IllegalArgumentException if the key is null + * @implNote The choice of when to push the changes to the data store is left up to the + * implementation. * @apiNote Put must have the following properties: - *

    - *
  1. Creates a new entry in the cache, if the key-value pair does - * not exist in the cache or underlying data store.
    2. - *
  2. Updates the entry in the cache when the key-value pair - * already exists. - *
  3. Deletes the entry when given a {@code null} value.
    4. - *
+ *
    + *
  1. Creates a new entry in the cache, if the key-value pair does not exist in the cache + * or underlying data store. + *
  2. Updates the entry in the cache when the key-value pair already exists. + *
  3. Deletes the entry when given a {@code null} value. + *
*/ void put(K k, V v); /** - * Delete an entry from the cache, marking it for deletion inside the data - * store. Makes no guarantees about when the value is actually deleted from - * the underlying data store. + * Delete an entry from the cache, marking it for deletion inside the data store. Makes no + * guarantees about when the value is actually deleted from the underlying data store. * - * @param k - * the key of the entry to be deleted - * @throws RuntimeException - * if the underlying data store is closed - * @throws IllegalArgumentException - * if the key is null - * @implNote The choice of when to push the changes to the data store is - * left up to the implementation. + * @param k the key of the entry to be deleted + * @throws RuntimeException if the underlying data store is closed + * @throws IllegalArgumentException if the key is null + * @implNote The choice of when to push the changes to the data store is left up to the + * implementation. */ void delete(K k); @@ -138,37 +119,32 @@ public interface IKeyValueStore extends AutoCloseable { // --------------------------------------------------------------------------------------------------- /** - * Puts or updates the data store with the given key-value pairs, as - * follows: + * Puts or updates the data store with the given key-value pairs, as follows: + * *
    - *
  • if the key is present in the data store, the stored - * value is overwritten
    • - *
  • if the key is not present in the data store, the new - * key-value pair is stored
    • - *
  • if the value is null, the matching stored key will be - * deleted from the data store, or
    • + *
  • if the key is present in the data store, the stored value is overwritten + *
  • if the key is not present in the data store, the new key-value pair is + * stored + *
  • if the value is null, the matching stored key will be deleted from the + * data store, or *
* - * @param inputMap - * a {@link Map} of key-value pairs to be updated in the database - * @throws RuntimeException - * if the data store is closed - * @throws IllegalArgumentException - * if the map contains a null key + * @param inputMap a {@link Map} of key-value pairs to be updated in the database + * @throws RuntimeException if the data store is closed + * @throws IllegalArgumentException if the map contains a null key */ void putBatch(Map inputMap); - void putToBatch(byte[] key, byte[] value); + void putToBatch(K key, V value); + void commitBatch(); /** * Similar to delete, except operates on a list of keys * * @param keys - * @throws RuntimeException - * if the data store is closed - * @throws IllegalArgumentException - * if the collection contains a null key + * @throws RuntimeException if the data store is closed + * @throws IllegalArgumentException if the collection contains a null key */ void deleteBatch(Collection keys); } From e40b69903dca5de2dda389527d89f613fe875760 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Fri, 18 May 2018 13:53:34 -0400 Subject: [PATCH 14/35] new data source class that deletes from the current db only keys not present in the archive db --- .../org/aion/mcf/ds/ArchivedDataSource.java | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 modMcf/src/org/aion/mcf/ds/ArchivedDataSource.java diff --git a/modMcf/src/org/aion/mcf/ds/ArchivedDataSource.java b/modMcf/src/org/aion/mcf/ds/ArchivedDataSource.java new file mode 100644 index 0000000000..14ce977950 --- /dev/null +++ b/modMcf/src/org/aion/mcf/ds/ArchivedDataSource.java @@ -0,0 +1,125 @@ +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: + * Aion foundation. + ******************************************************************************/ +package org.aion.mcf.ds; + +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import org.aion.base.db.IByteArrayKeyValueDatabase; +import org.aion.base.db.IByteArrayKeyValueStore; + +/** + * A data source with archived data that must no be deleted. + * + * @author Alexandra Roatis + */ +public class ArchivedDataSource implements IByteArrayKeyValueStore { + + IByteArrayKeyValueDatabase data, archive; + + public ArchivedDataSource(IByteArrayKeyValueDatabase _db, IByteArrayKeyValueDatabase _archive) { + this.data = _db; + this.archive = _archive; + } + + @Override + public boolean isEmpty() { + return data.isEmpty(); + } + + @Override + public Set keys() { + return data.keys(); + } + + @Override + public Optional get(byte[] key) { + return data.get(key); + } + + @Override + public void put(byte[] key, byte[] value) { + if (value != null) { + data.put(key, value); + } else { + // internal delete will check if archived + delete(key); + } + } + + @Override + public void delete(byte[] key) { + // delete key only if not archived + if (!archive.get(key).isPresent()) { + data.delete(key); + } + } + + @Override + public void putBatch(Map batch) { + for (Map.Entry entry : batch.entrySet()) { + // will check if archived + putToBatch(entry.getKey(), entry.getValue()); + } + commitBatch(); + } + + @Override + public void putToBatch(byte[] key, byte[] value) { + if (value != null) { + data.putToBatch(key, value); + } else { + // deleted key only if not archived + if (!archive.get(key).isPresent()) { + data.putToBatch(key, null); + } + } + } + + @Override + public void commitBatch() { + data.commitBatch(); + } + + @Override + public void deleteBatch(Collection keys) { + for (byte[] key : keys) { + // will check if archived + putToBatch(key, null); + } + commitBatch(); + } + + @Override + public void close() { + data.close(); + archive.close(); + } +} From 471614e2b34caf3cd19e98be6864379c73bdc58e Mon Sep 17 00:00:00 2001 From: Alexandra Date: Fri, 18 May 2018 14:25:52 -0400 Subject: [PATCH 15/35] refactored journal prune class to take a data store as argument; method to check if open database added to data store interface --- .../src/org/aion/base/db/IKeyValueStore.java | 9 +++ .../aion/db/generic/DatabaseWithCache.java | 12 +-- .../org/aion/db/generic/LockedDatabase.java | 13 ++++ .../org/aion/db/generic/TimedDatabase.java | 9 +++ .../src/org/aion/db/impl/AbstractDB.java | 74 +++++++++---------- .../org/aion/mcf/ds/ArchivedDataSource.java | 6 ++ modMcf/src/org/aion/mcf/ds/XorDataSource.java | 16 ++-- .../aion/mcf/trie/JournalPruneDataSource.java | 33 ++++----- 8 files changed, 99 insertions(+), 73 deletions(-) diff --git a/modAionBase/src/org/aion/base/db/IKeyValueStore.java b/modAionBase/src/org/aion/base/db/IKeyValueStore.java index 831d8257a6..d480cd619d 100644 --- a/modAionBase/src/org/aion/base/db/IKeyValueStore.java +++ b/modAionBase/src/org/aion/base/db/IKeyValueStore.java @@ -147,4 +147,13 @@ public interface IKeyValueStore extends AutoCloseable { * @throws IllegalArgumentException if the collection contains a null key */ void deleteBatch(Collection keys); + + /** + * Checks that the data store connection is open. Throws a {@link RuntimeException} if the data + * store connection is closed. + * + * @implNote Always do this check after acquiring a lock on the class/data. Otherwise it might + * produce inconsistent results due to lack of synchronization. + */ + void check(); } diff --git a/modDbImpl/src/org/aion/db/generic/DatabaseWithCache.java b/modDbImpl/src/org/aion/db/generic/DatabaseWithCache.java index e2380e64ef..80636044ae 100644 --- a/modDbImpl/src/org/aion/db/generic/DatabaseWithCache.java +++ b/modDbImpl/src/org/aion/db/generic/DatabaseWithCache.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -163,14 +163,8 @@ public CacheStats getStats() { return this.loadingCache.stats(); } - /** - * Checks that the database connection is open. - * Throws a {@link RuntimeException} if the database connection is closed. - * - * @implNote Always do this check after acquiring a lock on the class/data. - * Otherwise it might produce inconsistent results due to lack of synchronization. - */ - private void check() { + @Override + public void check() { if (!database.isOpen()) { throw new RuntimeException("Database is not opened: " + this); } diff --git a/modDbImpl/src/org/aion/db/generic/LockedDatabase.java b/modDbImpl/src/org/aion/db/generic/LockedDatabase.java index ad7ed79c6d..9015430f35 100644 --- a/modDbImpl/src/org/aion/db/generic/LockedDatabase.java +++ b/modDbImpl/src/org/aion/db/generic/LockedDatabase.java @@ -366,6 +366,19 @@ public void deleteBatch(Collection keys) { } } + @Override + public void check() { + // acquire read lock + lock.readLock().lock(); + + try { + database.check(); + } finally { + // releasing read lock + lock.readLock().unlock(); + } + } + @Override public void drop() { // acquire write lock diff --git a/modDbImpl/src/org/aion/db/generic/TimedDatabase.java b/modDbImpl/src/org/aion/db/generic/TimedDatabase.java index 4a70fddc23..aa0fcfe372 100644 --- a/modDbImpl/src/org/aion/db/generic/TimedDatabase.java +++ b/modDbImpl/src/org/aion/db/generic/TimedDatabase.java @@ -252,6 +252,15 @@ public void deleteBatch(Collection keys) { LOG.debug(database.toString() + " deleteBatch(" + keys.size() + ") in " + (t2 - t1) + " ns."); } + @Override + public void check() { + long t1 = System.nanoTime(); + database.check(); + long t2 = System.nanoTime(); + + LOG.debug(database.toString() + " check() in " + (t2 - t1) + " ns."); + } + @Override public void drop() { long t1 = System.nanoTime(); diff --git a/modDbImpl/src/org/aion/db/impl/AbstractDB.java b/modDbImpl/src/org/aion/db/impl/AbstractDB.java index ee346f2ec8..9fa6243eed 100644 --- a/modDbImpl/src/org/aion/db/impl/AbstractDB.java +++ b/modDbImpl/src/org/aion/db/impl/AbstractDB.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -34,18 +34,16 @@ ******************************************************************************/ package org.aion.db.impl; -import org.aion.base.db.IByteArrayKeyValueDatabase; -import org.aion.base.util.ByteArrayWrapper; -import org.aion.log.AionLoggerFactory; -import org.aion.log.LogEnum; -import org.h2.store.fs.FileUtils; -import org.slf4j.Logger; - import java.io.File; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; import java.util.stream.Stream; +import org.aion.base.db.IByteArrayKeyValueDatabase; +import org.aion.base.util.ByteArrayWrapper; +import org.aion.log.AionLoggerFactory; +import org.aion.log.LogEnum; +import org.slf4j.Logger; /** * Common functionality for database implementations. @@ -70,7 +68,8 @@ protected AbstractDB(String name) { this.name = name; } - protected AbstractDB(String name, String path, boolean enableDbCache, boolean enableDbCompression) { + protected AbstractDB( + String name, String path, boolean enableDbCache, boolean enableDbCompression) { this(name); Objects.requireNonNull(path, "The database path cannot be null."); @@ -81,14 +80,22 @@ protected AbstractDB(String name, String path, boolean enableDbCache, boolean en } protected String propertiesInfo() { - return ""; // + return ""; // } @Override public boolean commit() { - // not implemented since we always commit the changes to the database for this implementation - throw new UnsupportedOperationException("Only automatic commits are supported by " + this.toString()); + // not implemented since we always commit the changes to the database for this + // implementation + throw new UnsupportedOperationException( + "Only automatic commits are supported by " + this.toString()); } @Override @@ -119,22 +126,16 @@ public Optional getPath() { return Optional.ofNullable(this.path); } - /** - * Checks that the database connection is open. - * Throws a {@link RuntimeException} if the database connection is closed. - * - * @implNote Always do this check after acquiring a lock on the class/data. - * Otherwise it might produce inconsistent results due to lack of synchronization. - */ - protected void check() { + @Override + public void check() { if (!isOpen()) { throw new RuntimeException("Database is not opened: " + this); } } /** - * Checks that the given key is not null. - * Throws a {@link IllegalArgumentException} if the key is null. + * Checks that the given key is not null. Throws a {@link IllegalArgumentException} if the key + * is null. */ public static void check(byte[] k) { if (k == null) { @@ -143,8 +144,8 @@ public static void check(byte[] k) { } /** - * Checks that the given collection of keys does not contain null values. - * Throws a {@link IllegalArgumentException} if a null key is present. + * Checks that the given collection of keys does not contain null values. Throws a {@link + * IllegalArgumentException} if a null key is present. */ public static void check(Collection keys) { if (keys.contains(null)) { @@ -170,23 +171,21 @@ public boolean isPersistent() { } /** - * For testing the lock functionality of public methods. - * Helps ensure that locks are released after normal or exceptional execution. + * For testing the lock functionality of public methods. Helps ensure that locks are released + * after normal or exceptional execution. * - * @return {@code true} when the resource is locked, - * {@code false} otherwise + * @return {@code true} when the resource is locked, {@code false} otherwise */ @Override public boolean isLocked() { return false; } - /** - * Functionality for directly interacting with the heap cache. - */ + /** Functionality for directly interacting with the heap cache. */ public abstract boolean commitCache(Map cache); - // IKeyValueStore functionality ------------------------------------------------------------------------------------ + // IKeyValueStore functionality + // ------------------------------------------------------------------------------------ @Override public Optional get(byte[] k) { @@ -200,12 +199,11 @@ public Optional get(byte[] k) { } /** - * Database specific get functionality, without locking required. Locking is applied in {@link #get(byte[])}. + * Database specific get functionality, without locking required. Locking is applied in {@link + * #get(byte[])}. * - * @param k - * the key for which the method must return the associated value + * @param k the key for which the method must return the associated value * @return the value stored in the database for the give key. */ protected abstract byte[] getInternal(byte[] k); - -} \ No newline at end of file +} diff --git a/modMcf/src/org/aion/mcf/ds/ArchivedDataSource.java b/modMcf/src/org/aion/mcf/ds/ArchivedDataSource.java index 14ce977950..9d0efcfcef 100644 --- a/modMcf/src/org/aion/mcf/ds/ArchivedDataSource.java +++ b/modMcf/src/org/aion/mcf/ds/ArchivedDataSource.java @@ -117,6 +117,12 @@ public void deleteBatch(Collection keys) { commitBatch(); } + @Override + public void check() { + data.check(); + archive.check(); + } + @Override public void close() { data.close(); diff --git a/modMcf/src/org/aion/mcf/ds/XorDataSource.java b/modMcf/src/org/aion/mcf/ds/XorDataSource.java index e4f77cbe52..f71259481d 100644 --- a/modMcf/src/org/aion/mcf/ds/XorDataSource.java +++ b/modMcf/src/org/aion/mcf/ds/XorDataSource.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -34,13 +34,11 @@ ******************************************************************************/ package org.aion.mcf.ds; -import org.aion.base.db.IByteArrayKeyValueDatabase; +import java.util.*; import org.aion.base.db.IByteArrayKeyValueStore; import org.aion.base.util.ByteArrayWrapper; import org.aion.base.util.ByteUtil; -import java.util.*; - public class XorDataSource implements IByteArrayKeyValueStore { IByteArrayKeyValueStore source; byte[] subKey; @@ -89,8 +87,9 @@ public void putBatch(Map rows) { } public void updateBatch(Map rows, boolean erasure) { - //not supported - throw new UnsupportedOperationException("ByteArrayWrapper map not supported in XorDataSource.updateBatch yet"); + // not supported + throw new UnsupportedOperationException( + "ByteArrayWrapper map not supported in XorDataSource.updateBatch yet"); } @Override @@ -104,6 +103,11 @@ public void deleteBatch(Collection keys) { } + @Override + public void check() { + source.check(); + } + @Override public boolean isEmpty() { // TODO Auto-generated method stub diff --git a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java index 31a1a1e9a1..5b1317ae3a 100644 --- a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java +++ b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java @@ -38,7 +38,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.base.db.IByteArrayKeyValueStore; import org.aion.base.util.ByteArrayWrapper; import org.aion.log.AionLoggerFactory; @@ -85,13 +84,13 @@ public String toString() { Map refCount = new HashMap<>(); - private IByteArrayKeyValueDatabase src; + private IByteArrayKeyValueStore src; // block hash => updates private LinkedHashMap blockUpdates = new LinkedHashMap<>(); private Updates currentUpdates = new Updates(); private AtomicBoolean enabled = new AtomicBoolean(true); - public JournalPruneDataSource(IByteArrayKeyValueDatabase src) { + public JournalPruneDataSource(IByteArrayKeyValueStore src) { this.src = src; } @@ -119,7 +118,7 @@ public void put(byte[] key, byte[] value) { src.put(key, value); } else { - checkOpen(); + check(); // Value does not exist, so we delete from current updates currentUpdates.deletedKeys.add(keyW); @@ -129,7 +128,7 @@ public void put(byte[] key, byte[] value) { if (value != null) { src.put(key, value); } else { - checkOpen(); + check(); } } } catch (Exception e) { @@ -152,7 +151,7 @@ public void delete(byte[] key) { lock.writeLock().lock(); try { - checkOpen(); + check(); currentUpdates.deletedKeys.add(ByteArrayWrapper.wrap(key)); // delete is delayed @@ -350,6 +349,8 @@ public void close() { try { src.close(); + } catch (Exception e) { + LOG.error("Could not close source due to ", e); } finally { lock.writeLock().unlock(); } @@ -375,7 +376,7 @@ public void deleteBatch(Collection keys) { lock.writeLock().lock(); try { - checkOpen(); + check(); // deletes are delayed keys.forEach(key -> currentUpdates.deletedKeys.add(ByteArrayWrapper.wrap(key))); @@ -397,7 +398,7 @@ public boolean isEmpty() { try { // the delayed deletes are not considered by this check until applied to the db if (!currentUpdates.insertedKeys.isEmpty()) { - checkOpen(); + check(); return false; } else { return src.isEmpty(); @@ -410,21 +411,13 @@ public boolean isEmpty() { } } - public IByteArrayKeyValueDatabase getSrc() { + public IByteArrayKeyValueStore getSrc() { return src; } - /** - * Checks that the data store connection is open. Throws a {@link RuntimeException} if the data - * store connection is closed. - * - * @implNote Always do this check after acquiring a lock on the class/data. Otherwise it might - * produce inconsistent results due to lack of synchronization. - */ - protected void checkOpen() { - if (!src.isOpen()) { - throw new RuntimeException("Data store is not opened: " + src); - } + @Override + public void check() { + src.check(); } /** From 9c5b5522d06217ca0f42c1551a446e1b12def1f2 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Fri, 18 May 2018 14:38:41 -0400 Subject: [PATCH 16/35] building the journal prune on top of the data source with archive --- .../org/aion/zero/impl/db/AionRepositoryImpl.java | 11 +++++++++++ modMcf/src/org/aion/mcf/config/CfgDb.java | 1 + modMcf/src/org/aion/mcf/db/AbstractRepository.java | 12 +++++++++++- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java index f1195566bb..231b07f353 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java +++ b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java @@ -542,6 +542,7 @@ public IRepository getSnapshotTo(byte[] root) { repo.blockStore = blockStore; repo.cfg = cfg; repo.stateDatabase = this.stateDatabase; + repo.stateWithArchive = this.stateWithArchive; repo.stateDSPrune = this.stateDSPrune; // pruning config @@ -624,6 +625,16 @@ public void close() { LOGGEN.error("Exception occurred while closing the state database.", e); } + try { + if (stateArchiveDatabase != null) { + stateArchiveDatabase.close(); + LOGGEN.info("State archive database closed."); + stateArchiveDatabase = null; + } + } catch (Exception e) { + LOGGEN.error("Exception occurred while closing the state archive database.", e); + } + try { if (transactionDatabase != null) { transactionDatabase.close(); diff --git a/modMcf/src/org/aion/mcf/config/CfgDb.java b/modMcf/src/org/aion/mcf/config/CfgDb.java index b1272ddccc..3ec6abc924 100644 --- a/modMcf/src/org/aion/mcf/config/CfgDb.java +++ b/modMcf/src/org/aion/mcf/config/CfgDb.java @@ -50,6 +50,7 @@ public static class Names { public static final String STORAGE = "storage"; public static final String STATE = "state"; + public static final String STATE_ARCHIVE = "stateArchive"; public static final String TRANSACTION = "transaction"; public static final String TX_CACHE = "pendingtxCache"; diff --git a/modMcf/src/org/aion/mcf/db/AbstractRepository.java b/modMcf/src/org/aion/mcf/db/AbstractRepository.java index 15246e6c98..70c4afd00b 100644 --- a/modMcf/src/org/aion/mcf/db/AbstractRepository.java +++ b/modMcf/src/org/aion/mcf/db/AbstractRepository.java @@ -40,6 +40,7 @@ import org.aion.mcf.config.CfgDb; import org.aion.mcf.core.AccountState; import org.aion.mcf.db.exception.InvalidFilePathException; +import org.aion.mcf.ds.ArchivedDataSource; import org.aion.mcf.trie.JournalPruneDataSource; import org.aion.mcf.trie.Trie; import org.aion.mcf.types.AbstractBlock; @@ -70,6 +71,7 @@ public abstract class AbstractRepository< protected static final String DETAILS_DB = CfgDb.Names.DETAILS; protected static final String STORAGE_DB = CfgDb.Names.STORAGE; protected static final String STATE_DB = CfgDb.Names.STATE; + protected static final String STATE_ARCHIVE_DB = CfgDb.Names.STATE_ARCHIVE; protected static final String PENDING_TX_POOL_DB = CfgDb.Names.TX_POOL; protected static final String PENDING_TX_CACHE_DB = CfgDb.Names.TX_CACHE; @@ -88,11 +90,13 @@ public abstract class AbstractRepository< protected IByteArrayKeyValueDatabase indexDatabase; protected IByteArrayKeyValueDatabase blockDatabase; protected IByteArrayKeyValueDatabase stateDatabase; + protected IByteArrayKeyValueDatabase stateArchiveDatabase; protected IByteArrayKeyValueDatabase txPoolDatabase; protected IByteArrayKeyValueDatabase pendingTxCacheDatabase; protected Collection databaseGroup; + protected ArchivedDataSource stateWithArchive; protected JournalPruneDataSource stateDSPrune; protected DetailsDataStore detailsDS; @@ -190,6 +194,11 @@ protected void initializeDatabasesAndCaches() throws Exception { this.stateDatabase = connectAndOpen(sharedProps); databaseGroup.add(stateDatabase); + // using state config for state_archive + sharedProps.setProperty(Props.DB_NAME, STATE_ARCHIVE_DB); + this.stateArchiveDatabase = connectAndOpen(sharedProps); + databaseGroup.add(stateArchiveDatabase); + // getting transaction specific properties sharedProps = cfg.getDatabaseConfig(TRANSACTION_DB); sharedProps.setProperty(Props.ENABLE_LOCKING, "false"); @@ -248,7 +257,8 @@ protected void initializeDatabasesAndCaches() throws Exception { // Setup the cache for transaction data source. this.detailsDS = new DetailsDataStore<>(detailsDatabase, storageDatabase, this.cfg); - stateDSPrune = new JournalPruneDataSource(stateDatabase); + stateWithArchive = new ArchivedDataSource(stateDatabase, stateArchiveDatabase); + stateDSPrune = new JournalPruneDataSource(stateWithArchive); // pruning config pruneEnabled = this.cfg.getPruneConfig().isEnabled(); From 8e487b5e7bce9d8940e6665c87a1ed731593a725 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Fri, 18 May 2018 15:10:51 -0400 Subject: [PATCH 17/35] archiving the world state at runtime --- .../aion/zero/impl/db/AionRepositoryImpl.java | 5 + .../org/aion/mcf/ds/ArchivedDataSource.java | 4 + .../aion/mcf/trie/JournalPruneDataSource.java | 16 + modMcf/src/org/aion/mcf/trie/Trie.java | 71 ++-- modMcf/src/org/aion/mcf/trie/TrieImpl.java | 344 ++++++++++++------ .../aion/mcf/trie/scan/ExtractToDatabase.java | 51 +++ 6 files changed, 341 insertions(+), 150 deletions(-) create mode 100644 modMcf/src/org/aion/mcf/trie/scan/ExtractToDatabase.java diff --git a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java index 231b07f353..f047665c3a 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java +++ b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java @@ -502,6 +502,11 @@ public void commitBlock(A0BlockHeader blockHeader) { detailsDS.syncLargeStorage(); if (pruneEnabled) { + if (blockHeader.getNumber() % archiveRate == 0 && stateDSPrune.isArchiveEnabled()) { + // archive block + worldState.saveDiffStateToDatabase( + blockHeader.getStateRoot(), stateDSPrune.getArchiveSource()); + } stateDSPrune.storeBlockChanges(blockHeader.getHash(), blockHeader.getNumber()); detailsDS .getStorageDSPrune() diff --git a/modMcf/src/org/aion/mcf/ds/ArchivedDataSource.java b/modMcf/src/org/aion/mcf/ds/ArchivedDataSource.java index 9d0efcfcef..0f0f35d2fa 100644 --- a/modMcf/src/org/aion/mcf/ds/ArchivedDataSource.java +++ b/modMcf/src/org/aion/mcf/ds/ArchivedDataSource.java @@ -128,4 +128,8 @@ public void close() { data.close(); archive.close(); } + + public IByteArrayKeyValueDatabase getArchiveDatabase() { + return archive; + } } diff --git a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java index 5b1317ae3a..3141f79d65 100644 --- a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java +++ b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java @@ -38,10 +38,12 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.base.db.IByteArrayKeyValueStore; import org.aion.base.util.ByteArrayWrapper; import org.aion.log.AionLoggerFactory; import org.aion.log.LogEnum; +import org.aion.mcf.ds.ArchivedDataSource; import org.slf4j.Logger; /** @@ -89,15 +91,21 @@ public String toString() { private LinkedHashMap blockUpdates = new LinkedHashMap<>(); private Updates currentUpdates = new Updates(); private AtomicBoolean enabled = new AtomicBoolean(true); + private final boolean hasArchive; public JournalPruneDataSource(IByteArrayKeyValueStore src) { this.src = src; + this.hasArchive = src instanceof ArchivedDataSource; } public void setPruneEnabled(boolean _enabled) { enabled.set(_enabled); } + public boolean isArchiveEnabled() { + return hasArchive; + } + public void put(byte[] key, byte[] value) { checkNotNull(key); @@ -415,6 +423,14 @@ public IByteArrayKeyValueStore getSrc() { return src; } + public IByteArrayKeyValueDatabase getArchiveSource() { + if (!hasArchive) { + return null; + } else { + return ((ArchivedDataSource) src).getArchiveDatabase(); + } + } + @Override public void check() { src.check(); diff --git a/modMcf/src/org/aion/mcf/trie/Trie.java b/modMcf/src/org/aion/mcf/trie/Trie.java index 8a0f865c30..34b38e312b 100644 --- a/modMcf/src/org/aion/mcf/trie/Trie.java +++ b/modMcf/src/org/aion/mcf/trie/Trie.java @@ -1,36 +1,45 @@ -/******************************************************************************* +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. * - * Copyright (c) 2017, 2018 Aion foundation. + * This file is part of the aion network project. * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see + * along with the aion network project source files. + * If not, see . * - * Contributors: + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: * Aion foundation. *******************************************************************************/ package org.aion.mcf.trie; +import org.aion.base.db.IByteArrayKeyValueDatabase; + /** - * Trie interface for the main data structure in Ethereum - * which is used to store both the account state and storage of each account. + * Trie interface for the main data structure in Ethereum which is used to store both the account + * state and storage of each account. */ public interface Trie { /** * Gets a value from the trie for a given key * - * @param key - * - any length byte array + * @param key - any length byte array * @return an rlp encoded byte array of the stored object */ byte[] get(byte[] key); @@ -38,18 +47,15 @@ public interface Trie { /** * Insert or update a value in the trie for a specified key * - * @param key - * - any length byte array - * @param value - * rlp encoded byte array of the object to store + * @param key - any length byte array + * @param value rlp encoded byte array of the object to store */ void update(byte[] key, byte[] value); /** * Deletes a key/value from the trie for a given key * - * @param key - * - any length byte array + * @param key - any length byte array */ void delete(byte[] key); @@ -63,37 +69,36 @@ public interface Trie { /** * Set the top node of the trie * - * @param root - * - 32-byte SHA-3 hash of the root node + * @param root - 32-byte SHA-3 hash of the root node */ void setRoot(byte[] root); /** * Used to check for corruption in the database. * - * @param root - * a world state trie root + * @param root a world state trie root * @return {@code true} if the root is valid, {@code false} otherwise */ boolean isValidRoot(byte[] root); - /** - * Commit all the changes until now - */ + /** Commit all the changes until now */ void sync(); void sync(boolean flushCache); - /** - * Discard all the changes until now - */ + /** Discard all the changes until now */ @Deprecated void undo(); String getTrieDump(); + String getTrieDump(byte[] stateRoot); + int getTrieSize(byte[] stateRoot); boolean validate(); -} \ No newline at end of file + long saveFullStateToDatabase(byte[] stateRoot, IByteArrayKeyValueDatabase db); + + long saveDiffStateToDatabase(byte[] stateRoot, IByteArrayKeyValueDatabase db); +} diff --git a/modMcf/src/org/aion/mcf/trie/TrieImpl.java b/modMcf/src/org/aion/mcf/trie/TrieImpl.java index 7f459824f5..4b3ef2ff2b 100644 --- a/modMcf/src/org/aion/mcf/trie/TrieImpl.java +++ b/modMcf/src/org/aion/mcf/trie/TrieImpl.java @@ -1,77 +1,78 @@ -/******************************************************************************* +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. * - * Copyright (c) 2017, 2018 Aion foundation. + * This file is part of the aion network project. * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see + * along with the aion network project source files. + * If not, see . * - * Contributors: + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: * Aion foundation. *******************************************************************************/ package org.aion.mcf.trie; +import static java.util.Arrays.copyOfRange; +import static org.aion.base.util.ByteArrayWrapper.wrap; +import static org.aion.base.util.ByteUtil.EMPTY_BYTE_ARRAY; +import static org.aion.base.util.ByteUtil.matchingNibbleLength; +import static org.aion.crypto.HashUtil.EMPTY_TRIE_HASH; +import static org.aion.rlp.CompactEncoder.*; +import static org.aion.rlp.RLP.calcElementPrefixSize; +import static org.spongycastle.util.Arrays.concatenate; + +import java.io.*; +import java.util.*; +import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.base.db.IByteArrayKeyValueStore; import org.aion.base.util.ByteArrayWrapper; import org.aion.base.util.FastByteComparisons; import org.aion.base.util.Hex; import org.aion.crypto.HashUtil; -import org.aion.mcf.trie.scan.CollectFullSetOfNodes; -import org.aion.mcf.trie.scan.CountNodes; -import org.aion.mcf.trie.scan.ScanAction; -import org.aion.mcf.trie.scan.TraceAllNodes; +import org.aion.mcf.trie.scan.*; import org.aion.rlp.RLP; import org.aion.rlp.RLPItem; import org.aion.rlp.RLPList; import org.aion.rlp.Value; -import java.io.*; -import java.util.*; - -import static java.util.Arrays.copyOfRange; -import static org.aion.base.util.ByteArrayWrapper.wrap; -import static org.aion.base.util.ByteUtil.EMPTY_BYTE_ARRAY; -import static org.aion.base.util.ByteUtil.matchingNibbleLength; -import static org.aion.crypto.HashUtil.EMPTY_TRIE_HASH; -import static org.aion.rlp.CompactEncoder.*; -import static org.aion.rlp.RLP.calcElementPrefixSize; -import static org.spongycastle.util.Arrays.concatenate; - /** - * The modified Merkle Patricia tree (trie) provides a persistent data structure - * to map between arbitrary-length binary data (byte arrays). It is defined in - * terms of a mutable data structure to map between 256-bit binary fragments and - * arbitrary-length binary data, typically implemented as a database. The core - * of the trie, and its sole requirement in terms of the protocol specification - * is to provide a single value that identifies a given set of key-value pairs, - * which may either a 32 byte sequence or the empty byte sequence. It is left as - * an implementation consideration to store and maintain the structure of the - * trie in a manner the allows effective and efficient realisation of the - * protocol. - *

- * The trie implements a caching mechanism and will use cached values if they - * are present. If a node is not present in the cache it will try to fetch it - * from the database and store the cached value. - *

- * Note: the data isn't persisted unless `sync` is explicitly called. - *

- * This Trie implementation supports node pruning (i.e. obsolete nodes are - * marked for removal in the Cache and actually removed from the underlying - * storage on [sync] call), but the algorithm is not suitable for the most - * general case. In general case a trie node might be referenced from several - * parent nodes and for correct pruning the reference counting algorithm needs - * to be implemented. As soon as the real life tree keys are hashes it is very - * unlikely the case so the pruning algorithm is simplified in this - * implementation. + * The modified Merkle Patricia tree (trie) provides a persistent data structure to map between + * arbitrary-length binary data (byte arrays). It is defined in terms of a mutable data structure to + * map between 256-bit binary fragments and arbitrary-length binary data, typically implemented as a + * database. The core of the trie, and its sole requirement in terms of the protocol specification + * is to provide a single value that identifies a given set of key-value pairs, which may either a + * 32 byte sequence or the empty byte sequence. It is left as an implementation consideration to + * store and maintain the structure of the trie in a manner the allows effective and efficient + * realisation of the protocol. + * + *

The trie implements a caching mechanism and will use cached values if they are present. If a + * node is not present in the cache it will try to fetch it from the database and store the cached + * value. + * + *

Note: the data isn't persisted unless `sync` is explicitly called. + * + *

This Trie implementation supports node pruning (i.e. obsolete nodes are marked for removal in + * the Cache and actually removed from the underlying storage on [sync] call), but the algorithm is + * not suitable for the most general case. In general case a trie node might be referenced from + * several parent nodes and for correct pruning the reference counting algorithm needs to be + * implemented. As soon as the real life tree keys are hashes it is very unlikely the case so the + * pruning algorithm is simplified in this implementation. * * @author Nick Savers * @since 20.05.2014 @@ -81,8 +82,7 @@ public class TrieImpl implements Trie { private static byte LIST_SIZE = 17; private static int MAX_SIZE = 20; - @Deprecated - private Object prevRoot; + @Deprecated private Object prevRoot; private Object root; private Cache cache; @@ -119,9 +119,7 @@ public Object getRoot() { return root; } - /** - * for testing TrieTest.testRollbackToRootScenarios - */ + /** for testing TrieTest.testRollbackToRootScenarios */ public void setRoot(Object root) { this.root = root; } @@ -152,9 +150,7 @@ public TrieImpl withPruningEnabled(boolean pruningEnabled) { return this; } - /** - * Retrieve a value from a key as String. - */ + /** Retrieve a value from a key as String. */ public byte[] get(String key) { return this.get(key.getBytes()); } @@ -169,9 +165,7 @@ public byte[] get(byte[] key) { } } - /** - * Insert key/value pair into trie. - */ + /** Insert key/value pair into trie. */ public void update(String key, String value) { this.update(key.getBytes(), value.getBytes()); } @@ -198,9 +192,7 @@ public synchronized boolean isValidRoot(byte[] root) { return !(this.getNode(root) == null); } - /** - * Delete a key/value pair from the trie. - */ + /** Delete a key/value pair from the trie. */ public void delete(String key) { this.update(key.getBytes(), EMPTY_BYTE_ARRAY); } @@ -215,8 +207,9 @@ public void delete(byte[] key) { @Override public byte[] getRootHash() { synchronized (cache) { - if (root == null || (root instanceof byte[] && ((byte[]) root).length == 0) || (root instanceof String && "" - .equals(root))) { + if (root == null + || (root instanceof byte[] && ((byte[]) root).length == 0) + || (root instanceof String && "".equals(root))) { return EMPTY_TRIE_HASH; } else if (root instanceof byte[]) { return (byte[]) this.getRoot(); @@ -241,8 +234,8 @@ private Object get(Object node, byte[] key) { byte[] k = unpackToNibbles(currentNode.get(0).asBytes()); Object v = currentNode.get(1).asObj(); - if (key.length - keypos >= k.length && Arrays - .equals(k, copyOfRange(key, keypos, k.length + keypos))) { + if (key.length - keypos >= k.length + && Arrays.equals(k, copyOfRange(key, keypos, k.length + keypos))) { node = v; keypos += k.length; } else { @@ -277,7 +270,7 @@ private Object insert(Object node, byte[] key, Object value) { } if (isEmptyNode(node)) { - Object[] newNode = new Object[] { packNibbles(key), value }; + Object[] newNode = new Object[] {packNibbles(key), value}; return this.putToCache(newNode); } @@ -295,7 +288,7 @@ private Object insert(Object node, byte[] key, Object value) { // Matching key pair (ie. there's already an object with this key) if (Arrays.equals(k, key)) { - Object[] newNode = new Object[] { packNibbles(key), value }; + Object[] newNode = new Object[] {packNibbles(key), value}; return this.putToCache(newNode); } @@ -311,7 +304,8 @@ private Object insert(Object node, byte[] key, Object value) { // Expand the 2 length slice to a 17 length slice // Create two nodes to putToCache into the new 17 length node Object oldNode = this.insert("", copyOfRange(k, matchingLength + 1, k.length), v); - Object newNode = this.insert("", copyOfRange(key, matchingLength + 1, key.length), value); + Object newNode = + this.insert("", copyOfRange(key, matchingLength + 1, key.length), value); // Create an expanded slice Object[] scaledSlice = emptyStringSlice(17); @@ -328,7 +322,8 @@ private Object insert(Object node, byte[] key, Object value) { // End of the chain, return return newHash; } else { - Object[] newNode = new Object[] { packNibbles(copyOfRange(key, 0, matchingLength)), newHash }; + Object[] newNode = + new Object[] {packNibbles(copyOfRange(key, 0, matchingLength)), newHash}; return this.putToCache(newNode); } } else { @@ -337,10 +332,15 @@ private Object insert(Object node, byte[] key, Object value) { Object[] newNode = copyNode(currentNode); // Replace the first nibble in the key - newNode[key[0]] = this.insert(currentNode.get(key[0]).asObj(), copyOfRange(key, 1, key.length), value); - - if (!FastByteComparisons - .equal(HashUtil.h256(getNode(newNode).encode()), HashUtil.h256(currentNode.encode()))) { + newNode[key[0]] = + this.insert( + currentNode.get(key[0]).asObj(), + copyOfRange(key, 1, key.length), + value); + + if (!FastByteComparisons.equal( + HashUtil.h256(getNode(newNode).encode()), + HashUtil.h256(currentNode.encode()))) { markRemoved(HashUtil.h256(currentNode.encode())); if (!isEmptyNode(currentNode.get(key[0]))) { markRemoved(currentNode.get(key[0]).asBytes()); @@ -379,9 +379,9 @@ private Object delete(Object node, byte[] key) { Object newNode; if (child.length() == PAIR_SIZE) { byte[] newKey = concatenate(k, unpackToNibbles(child.get(0).asBytes())); - newNode = new Object[] { packNibbles(newKey), child.get(1).asObj() }; + newNode = new Object[] {packNibbles(newKey), child.get(1).asObj()}; } else { - newNode = new Object[] { currentNode.get(0), hash }; + newNode = new Object[] {currentNode.get(0), hash}; } markRemoved(HashUtil.h256(currentNode.encode())); return this.putToCache(newNode); @@ -408,21 +408,22 @@ private Object delete(Object node, byte[] key) { Object[] newNode = null; if (amount == 16) { - newNode = new Object[] { packNibbles(new byte[] { 16 }), itemList[amount] }; + newNode = new Object[] {packNibbles(new byte[] {16}), itemList[amount]}; } else if (amount >= 0) { Value child = this.getNode(itemList[amount]); if (child.length() == PAIR_SIZE) { - key = concatenate(new byte[] { amount }, unpackToNibbles(child.get(0).asBytes())); - newNode = new Object[] { packNibbles(key), child.get(1).asObj() }; + key = concatenate(new byte[] {amount}, unpackToNibbles(child.get(0).asBytes())); + newNode = new Object[] {packNibbles(key), child.get(1).asObj()}; } else if (child.length() == LIST_SIZE) { - newNode = new Object[] { packNibbles(new byte[] { amount }), itemList[amount] }; + newNode = new Object[] {packNibbles(new byte[] {amount}), itemList[amount]}; } } else { newNode = itemList; } - if (!FastByteComparisons - .equal(HashUtil.h256(getNode(newNode).encode()), HashUtil.h256(currentNode.encode()))) { + if (!FastByteComparisons.equal( + HashUtil.h256(getNode(newNode).encode()), + HashUtil.h256(currentNode.encode()))) { markRemoved(HashUtil.h256(currentNode.encode())); } @@ -437,8 +438,8 @@ private void markRemoved(byte[] hash) { } /** - * Helper method to retrieve the actual node. If the node is not a list and - * length is > 32 bytes get the actual node from the db. + * Helper method to retrieve the actual node. If the node is not a list and length is > 32 bytes + * get the actual node from the db. */ private Value getNode(Object node) { @@ -465,7 +466,9 @@ private Object putToCache(Object node) { private boolean isEmptyNode(Object node) { Value n = new Value(node); - return (node == null || (n.isString() && (n.asString().isEmpty() || n.get(0).isNull())) || n.length() == 0); + return (node == null + || (n.isString() && (n.asString().isEmpty() || n.get(0).isNull())) + || n.length() == 0); } private Object[] copyNode(Value currentNode) { @@ -485,7 +488,8 @@ public boolean equals(Object trie) { if (this == trie) { return true; } - return trie instanceof Trie && Arrays.equals(this.getRootHash(), ((Trie) trie).getRootHash()); + return trie instanceof Trie + && Arrays.equals(this.getRootHash(), ((Trie) trie).getRootHash()); } @Override @@ -524,10 +528,7 @@ public TrieImpl copy() { } } - /** - * ****************************** Utility functions * - * ***************************** - */ + /** ****************************** Utility functions * ***************************** */ // Created an array of empty elements of required length private static Object[] emptyStringSlice(int l) { Object[] slice = new Object[l]; @@ -538,13 +539,12 @@ private static Object[] emptyStringSlice(int l) { } /** - * Insert/delete operations on a Trie structure leaves the old nodes in - * cache, this method scans the cache and removes them. The method is not - * thread safe, the tree should not be modified during the cleaning process. + * Insert/delete operations on a Trie structure leaves the old nodes in cache, this method scans + * the cache and removes them. The method is not thread safe, the tree should not be modified + * during the cleaning process. */ public void cleanCache() { synchronized (cache) { - CollectFullSetOfNodes collectAction = new CollectFullSetOfNodes(); this.scanTree(this.getRootHash(), collectAction); @@ -576,7 +576,6 @@ public void printFootPrint() { public void scanTree(byte[] hash, ScanAction scanAction) { synchronized (cache) { - Value node = this.getCache().get(hash); if (node == null) { throw new RuntimeException("Not found: " + Hex.toHexString(hash)); @@ -638,6 +637,57 @@ public void scanTreeLoop(byte[] hash, ScanAction scanAction) { } } + /** + * Scans the trie similar to {@link #scanTreeLoop(byte[], ScanAction)}, but stops once a state + * is found in the given database. + * + * @param hash state root + * @param scanAction action to perform on each node + * @param db database containing keys that need not be explored + */ + public void scanTreeDiffLoop( + byte[] hash, ScanAction scanAction, IByteArrayKeyValueDatabase db) { + + ArrayList hashes = new ArrayList<>(); + hashes.add(hash); + + while (!hashes.isEmpty()) { + synchronized (cache) { + byte[] myHash = hashes.remove(0); + Value node = this.getCache().get(myHash); + if (node == null) { + System.out.println("Skipped key. Not found: " + Hex.toHexString(myHash)); + } else { + if (node.isList()) { + List siblings = node.asList(); + if (siblings.size() == PAIR_SIZE) { + Value val = new Value(siblings.get(1)); + if (val.isHashCode() && !hasTerminator((byte[]) siblings.get(0))) { + // scanTree(val.asBytes(), scanAction); + byte[] valBytes = val.asBytes(); + if (!db.get(valBytes).isPresent()) { + hashes.add(valBytes); + } + } + } else { + for (int j = 0; j < LIST_SIZE; ++j) { + Value val = new Value(siblings.get(j)); + if (val.isHashCode()) { + // scanTree(val.asBytes(), scanAction); + byte[] valBytes = val.asBytes(); + if (!db.get(valBytes).isPresent()) { + hashes.add(valBytes); + } + } + } + } + scanAction.doOnNode(myHash, node); + } + } + } + } + } + public void deserialize(byte[] data) { synchronized (cache) { RLPList rlpList = (RLPList) RLP.decode2(data).get(0); @@ -695,21 +745,43 @@ public byte[] serialize() { byte[] keysHeader = RLP.encodeLongElementHeader(keysTotalSize); byte[] valsHeader = RLP.encodeListHeader(valsTotalSize); - byte[] listHeader = RLP.encodeListHeader( - keysTotalSize + keysHeader.length + valsTotalSize + valsHeader.length + root.length); - - byte[] rlpData = new byte[keysTotalSize + keysHeader.length + valsTotalSize + valsHeader.length - + listHeader.length + root.length]; + byte[] listHeader = + RLP.encodeListHeader( + keysTotalSize + + keysHeader.length + + valsTotalSize + + valsHeader.length + + root.length); + + byte[] rlpData = + new byte + [keysTotalSize + + keysHeader.length + + valsTotalSize + + valsHeader.length + + listHeader.length + + root.length]; // copy headers: // [ rlp_list_header, rlp_keys_header, rlp_keys, rlp_vals_header, // rlp_val] System.arraycopy(listHeader, 0, rlpData, 0, listHeader.length); System.arraycopy(keysHeader, 0, rlpData, listHeader.length, keysHeader.length); - System.arraycopy(valsHeader, 0, rlpData, (listHeader.length + keysHeader.length + keysTotalSize), + System.arraycopy( + valsHeader, + 0, + rlpData, + (listHeader.length + keysHeader.length + keysTotalSize), valsHeader.length); - System.arraycopy(root, 0, rlpData, - (listHeader.length + keysHeader.length + keysTotalSize + valsTotalSize + valsHeader.length), + System.arraycopy( + root, + 0, + rlpData, + (listHeader.length + + keysHeader.length + + keysTotalSize + + valsTotalSize + + valsHeader.length), root.length); int k_1 = 0; @@ -720,15 +792,26 @@ public byte[] serialize() { continue; } - System.arraycopy(key.getData(), 0, rlpData, (listHeader.length + keysHeader.length + k_1), + System.arraycopy( + key.getData(), + 0, + rlpData, + (listHeader.length + keysHeader.length + k_1), key.getData().length); k_1 += key.getData().length; byte[] valBytes = RLP.encodeElement(node.getValue().getData()); - System.arraycopy(valBytes, 0, rlpData, - listHeader.length + keysHeader.length + keysTotalSize + valsHeader.length + k_2, + System.arraycopy( + valBytes, + 0, + rlpData, + listHeader.length + + keysHeader.length + + keysTotalSize + + valsHeader.length + + k_2, valBytes.length); k_2 += valBytes.length; } @@ -793,12 +876,14 @@ public boolean validate() { synchronized (cache) { final int[] cnt = new int[1]; try { - scanTree(getRootHash(), new ScanAction() { - @Override - public void doOnNode(byte[] hash, Value node) { - cnt[0]++; - } - }); + scanTree( + getRootHash(), + new ScanAction() { + @Override + public void doOnNode(byte[] hash, Value node) { + cnt[0]++; + } + }); } catch (Exception e) { return false; } @@ -806,4 +891,29 @@ public void doOnNode(byte[] hash, Value node) { } } + @Override + public long saveFullStateToDatabase(byte[] stateRoot, IByteArrayKeyValueDatabase db) { + ExtractToDatabase traceAction = new ExtractToDatabase(db); + traceTrie(stateRoot, traceAction); + return traceAction.count; + } + + private void traceDiffTrie(byte[] stateRoot, ScanAction action, IByteArrayKeyValueDatabase db) { + synchronized (cache) { + Value value = new Value(stateRoot); + + if (value.isHashCode() && !db.get(value.asBytes()).isPresent()) { + scanTreeDiffLoop(stateRoot, action, db); + } else { + action.doOnNode(stateRoot, value); + } + } + } + + @Override + public long saveDiffStateToDatabase(byte[] stateRoot, IByteArrayKeyValueDatabase db) { + ExtractToDatabase traceAction = new ExtractToDatabase(db); + traceDiffTrie(stateRoot, traceAction, db); + return traceAction.count; + } } diff --git a/modMcf/src/org/aion/mcf/trie/scan/ExtractToDatabase.java b/modMcf/src/org/aion/mcf/trie/scan/ExtractToDatabase.java new file mode 100644 index 0000000000..bb617f82f5 --- /dev/null +++ b/modMcf/src/org/aion/mcf/trie/scan/ExtractToDatabase.java @@ -0,0 +1,51 @@ +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: + * Aion foundation. + ******************************************************************************/ +package org.aion.mcf.trie.scan; + +import org.aion.base.db.IByteArrayKeyValueDatabase; +import org.aion.rlp.Value; + +/** @author Alexandra Roatis */ +public class ExtractToDatabase implements ScanAction { + + // only the keys are relevant so the value will be this constant + byte[] dummy_value = new byte[] {0}; + IByteArrayKeyValueDatabase db; + public long count = 0; + + public ExtractToDatabase(IByteArrayKeyValueDatabase _db) { + this.db = _db; + } + + @Override + public void doOnNode(byte[] hash, Value node) { + db.put(hash, dummy_value); + count++; + } +} From 631da9ef9e2a69274a9b010fdef9551296feab02 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Fri, 18 May 2018 16:23:29 -0400 Subject: [PATCH 18/35] bugfix: incorrect variable update --- modMcf/src/org/aion/mcf/config/CfgPrune.java | 23 +++++++++++--------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/modMcf/src/org/aion/mcf/config/CfgPrune.java b/modMcf/src/org/aion/mcf/config/CfgPrune.java index 4e71fed6e9..0fb56f98e9 100644 --- a/modMcf/src/org/aion/mcf/config/CfgPrune.java +++ b/modMcf/src/org/aion/mcf/config/CfgPrune.java @@ -44,16 +44,19 @@ public class CfgPrune implements IPruneConfig { private int current; private int archive; + private static final int MINIMUM_CURRENT_COUNT = 128; + private static final int MINIMUM_ARCHIVE_RATE = 1000; + public CfgPrune() { this.enabled = true; - this.current = 128; - this.archive = 10000; + this.current = MINIMUM_CURRENT_COUNT; + this.archive = MINIMUM_ARCHIVE_RATE; } public CfgPrune(boolean _enabled) { this.enabled = _enabled; - this.current = 128; - this.archive = 10000; + this.current = MINIMUM_CURRENT_COUNT; + this.archive = MINIMUM_ARCHIVE_RATE; } public void fromXML(final XMLStreamReader sr) throws XMLStreamException { @@ -69,16 +72,16 @@ public void fromXML(final XMLStreamReader sr) throws XMLStreamException { break; case "current_count": this.current = Integer.parseInt(Cfg.readValue(sr)); - // must be at least 128 - if (this.current < 128) { - this.current = 128; + // must be at least MINIMUM_CURRENT_COUNT + if (this.current < MINIMUM_CURRENT_COUNT) { + this.current = MINIMUM_CURRENT_COUNT; } break; case "archive_rate": this.archive = Integer.parseInt(Cfg.readValue(sr)); - // must be at least 1000 - if (this.current < 1000) { - this.current = 1000; + // must be at least MINIMUM_ARCHIVE_RATE + if (this.archive < MINIMUM_ARCHIVE_RATE) { + this.archive = MINIMUM_ARCHIVE_RATE; } break; default: From 2e89e3f3543c809d2d73948cbbfe618d70de4ee6 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Wed, 23 May 2018 16:46:39 -0400 Subject: [PATCH 19/35] bugfix: reseting the root back to its initial value after recovery --- .../src/org/aion/zero/impl/AionBlockchainImpl.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/modAionImpl/src/org/aion/zero/impl/AionBlockchainImpl.java b/modAionImpl/src/org/aion/zero/impl/AionBlockchainImpl.java index dfa660e1d3..650bcebcba 100644 --- a/modAionImpl/src/org/aion/zero/impl/AionBlockchainImpl.java +++ b/modAionImpl/src/org/aion/zero/impl/AionBlockchainImpl.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -1356,6 +1356,9 @@ public synchronized boolean recoverWorldState(IRepository repository, AionBlock AionRepositoryImpl repo = (AionRepositoryImpl) repository; + // keeping track of the original root + byte[] originalRoot = repo.getRoot(); + Deque dirtyBlocks = new ArrayDeque<>(); // already known to be missing the state dirtyBlocks.push(block); @@ -1397,6 +1400,9 @@ public synchronized boolean recoverWorldState(IRepository repository, AionBlock // update the repository repo.flush(); + // setting the root back to its correct value + repo.syncToRoot(originalRoot); + // return a flag indicating if the recovery worked return repo.isValidRoot(block.getStateRoot()); } From f8242dbbe22a4f16305b57140c597aab3cd0916c Mon Sep 17 00:00:00 2001 From: Alexandra Date: Wed, 23 May 2018 16:49:49 -0400 Subject: [PATCH 20/35] optimizing the sync BACKWARD and FORWARD behavior --- .../aion/zero/impl/sync/TaskImportBlocks.java | 46 +++++++++++++------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java b/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java index bb7577a2d2..e69aa9ae3e 100644 --- a/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java +++ b/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java @@ -29,6 +29,11 @@ package org.aion.zero.impl.sync; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; import org.aion.base.util.ByteArrayWrapper; import org.aion.mcf.core.ImportResult; import org.aion.p2p.IP2pMgr; @@ -36,16 +41,12 @@ import org.aion.zero.impl.types.AionBlock; import org.slf4j.Logger; -import java.util.List; -import java.util.Map; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - /** - * @author chris * handle process of importing blocks to repo - * TODO: targeted send + * + *

TODO: targeted send + * + * @author chris */ final class TaskImportBlocks implements Runnable { @@ -73,8 +74,7 @@ final class TaskImportBlocks implements Runnable { final BlockingQueue downloadedBlocks, final Map importedBlockHashes, final Map peerStates, - final Logger log - ) { + final Logger log) { this.p2p = p2p; this.chain = _chain; this.start = _start; @@ -103,7 +103,8 @@ public void run() { PeerState state = peerStates.get(bw.getNodeIdHash()); if (state == null) { - log.warn("This is not supposed to happen, but the peer is sending us blocks without ask"); + log.warn( + "This is not supposed to happen, but the peer is sending us blocks without ask"); } for (AionBlock b : batch) { @@ -113,20 +114,24 @@ public void run() { importResult = this.chain.tryToConnect(b); } catch (Throwable e) { log.error(" {}", e.toString()); - if (e.getMessage() != null && e.getMessage().contains("No space left on device")) { + if (e.getMessage() != null + && e.getMessage().contains("No space left on device")) { log.error("Shutdown due to lack of disk space."); System.exit(0); } continue; } long t2 = System.currentTimeMillis(); - log.info("", + log.info( + "", bw.getDisplayId(), + state.getMode(), b.getShortHash(), b.getNumber(), b.getTransactionsList().size(), importResult, t2 - t1); + switch (importResult) { case IMPORTED_BEST: case IMPORTED_NOT_BEST: @@ -154,13 +159,20 @@ public void run() { // we found the fork point state.setMode(PeerState.Mode.FORWARD); state.setBase(lastBlock); - } else if (mode == PeerState.Mode.FORWARD) { // continue state.setBase(lastBlock); // if the imported best block, switch back to normal mode if (importResult == ImportResult.IMPORTED_BEST) { state.setMode(PeerState.Mode.NORMAL); + // switch peers to NORMAL otherwise they may never switch back + for (PeerState peerState : peerStates.values()) { + if (peerState.getMode() != PeerState.Mode.NORMAL) { + peerState.setMode(PeerState.Mode.NORMAL); + peerState.setBase(b.getNumber()); + peerState.resetLastHeaderRequest(); + } + } } } break; @@ -176,6 +188,12 @@ public void run() { break; } } + + // if any block results in NO_PARENT, all subsequent blocks will too + if (importResult == ImportResult.NO_PARENT) { + log.debug("Stopped importing batch due to NO_PARENT result."); + break; + } } if (state != null) { From cf03e5426ba64bbc659565e0f8eb1c7a5b95c9a5 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Wed, 23 May 2018 17:36:11 -0400 Subject: [PATCH 21/35] fixing config file --- modBoot/resource/config.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/modBoot/resource/config.xml b/modBoot/resource/config.xml index a3234cb7a4..b81265be81 100644 --- a/modBoot/resource/config.xml +++ b/modBoot/resource/config.xml @@ -73,8 +73,6 @@ 10000 - - 1000 leveldb From 47da91467d4fe3587c0dbe2b7efce30143afc6a2 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Thu, 24 May 2018 09:57:32 -0400 Subject: [PATCH 22/35] bugfix: printing the size of the trie states should only consider the main chain --- .../org/aion/zero/impl/db/RecoveryUtils.java | 97 ++++++++++++------- 1 file changed, 60 insertions(+), 37 deletions(-) diff --git a/modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java b/modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java index fa5590f630..7afe26f58c 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java +++ b/modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -22,6 +22,9 @@ ******************************************************************************/ package org.aion.zero.impl.db; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import org.aion.base.type.IBlock; import org.aion.log.AionLoggerFactory; import org.aion.mcf.db.IBlockStoreBase; @@ -30,19 +33,15 @@ import org.aion.zero.impl.core.IAionBlockchain; import org.aion.zero.impl.types.AionBlock; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - public class RecoveryUtils { public enum Status { - SUCCESS, FAILURE, ILLEGAL_ARGUMENT + SUCCESS, + FAILURE, + ILLEGAL_ARGUMENT } - /** - * Used by the CLI call. - */ + /** Used by the CLI call. */ public static Status revertTo(long nbBlock) { // ensure mining is disabled CfgAion cfg = CfgAion.inst(); @@ -66,9 +65,7 @@ public static Status revertTo(long nbBlock) { return status; } - /** - * Used by the CLI call. - */ + /** Used by the CLI call. */ public static void pruneAndCorrect() { // ensure mining is disabled CfgAion cfg = CfgAion.inst(); @@ -101,9 +98,7 @@ public static void pruneAndCorrect() { blockchain.getRepository().close(); } - /** - * Used by the CLI call. - */ + /** Used by the CLI call. */ public static void dbCompact() { // ensure mining is disabled CfgAion cfg = CfgAion.inst(); @@ -126,9 +121,7 @@ public static void dbCompact() { repository.close(); } - /** - * Used by the CLI call. - */ + /** Used by the CLI call. */ public static void dumpBlocks(long count) { // ensure mining is disabled CfgAion cfg = CfgAion.inst(); @@ -161,9 +154,7 @@ public static void dumpBlocks(long count) { repository.close(); } - /** - * Used by internal world state recovery method. - */ + /** Used by internal world state recovery method. */ public static Status revertTo(IAionBlockchain blockchain, long nbBlock) { IBlockStoreBase store = blockchain.getBlockStore(); @@ -175,12 +166,15 @@ public static Status revertTo(IAionBlockchain blockchain, long nbBlock) { long nbBestBlock = bestBlock.getNumber(); - System.out.println("Attempting to revert best block from " + nbBestBlock + " to " + nbBlock + " ..."); + System.out.println( + "Attempting to revert best block from " + nbBestBlock + " to " + nbBlock + " ..."); // exit with warning if the given block is larger or negative if (nbBlock < 0) { System.out.println( - "Negative values <" + nbBlock + "> cannot be interpreted as block numbers. Nothing to do."); + "Negative values <" + + nbBlock + + "> cannot be interpreted as block numbers. Nothing to do."); return Status.ILLEGAL_ARGUMENT; } if (nbBestBlock == 0) { @@ -189,13 +183,19 @@ public static Status revertTo(IAionBlockchain blockchain, long nbBlock) { } if (nbBlock == nbBestBlock) { System.out.println( - "The block " + nbBlock + " is the current best block stored in the database. Nothing to do."); + "The block " + + nbBlock + + " is the current best block stored in the database. Nothing to do."); return Status.ILLEGAL_ARGUMENT; } if (nbBlock > nbBestBlock) { - System.out.println("The block #" + nbBlock + " is greater than the current best block #" + nbBestBlock - + " stored in the database. " - + "Cannot move to that block without synchronizing with peers. Start Aion instance to sync."); + System.out.println( + "The block #" + + nbBlock + + " is greater than the current best block #" + + nbBestBlock + + " stored in the database. " + + "Cannot move to that block without synchronizing with peers. Start Aion instance to sync."); return Status.ILLEGAL_ARGUMENT; } @@ -224,7 +224,7 @@ public static void printStateTrieSize(long blockNumber) { AionRepositoryImpl repository = AionRepositoryImpl.inst(); AionBlockStore store = repository.getBlockStore(); - long topBlock = store.getMaxNumber(); + long topBlock = store.getBestBlock().getNumber(); if (topBlock < 0) { System.out.println("The database is empty. Cannot print block information."); return; @@ -244,17 +244,33 @@ public static void printStateTrieSize(long blockNumber) { stateRoot = block.getStateRoot(); try { System.out.println( - "Block hash: " + block.getShortHash() + ", number: " + block.getNumber() + ", tx count: " - + block.getTransactionsList().size() + ", state trie kv count = " + repository - .getWorldState().getTrieSize(stateRoot)); + "Block hash: " + + block.getShortHash() + + ", number: " + + block.getNumber() + + ", tx count: " + + block.getTransactionsList().size() + + ", state trie kv count = " + + repository.getWorldState().getTrieSize(stateRoot)); } catch (RuntimeException e) { System.out.println( - "Block hash: " + block.getShortHash() + ", number: " + block.getNumber() + ", tx count: " - + block.getTransactionsList().size() + ", state trie kv count threw exception: " + e - .getMessage()); + "Block hash: " + + block.getShortHash() + + ", number: " + + block.getNumber() + + ", tx count: " + + block.getTransactionsList().size() + + ", state trie kv count threw exception: " + + e.getMessage()); } } else { - System.out.println("Null block found at level " + targetBlock + "."); + long count = store.getBlocksByNumber(targetBlock).size(); + System.out.println( + "Null block found at level " + + targetBlock + + ". There " + + (count == 1 ? "is 1 block" : "are " + count + " blocks") + + " at this level. No main chain block found."); } targetBlock++; } @@ -296,8 +312,15 @@ public static void printStateTrieDump(long blockNumber) { } byte[] stateRoot = block.getStateRoot(); - System.out.println("\nBlock hash: " + block.getShortHash() + ", number: " + blockNumber + ", tx count: " + block - .getTransactionsList().size() + "\n\n" + repository.getWorldState().getTrieDump(stateRoot)); + System.out.println( + "\nBlock hash: " + + block.getShortHash() + + ", number: " + + blockNumber + + ", tx count: " + + block.getTransactionsList().size() + + "\n\n" + + repository.getWorldState().getTrieDump(stateRoot)); repository.close(); } From b42dd5043b3db15e428555a4b7f1991ee68f4a7f Mon Sep 17 00:00:00 2001 From: Alexandra Date: Thu, 24 May 2018 10:39:06 -0400 Subject: [PATCH 23/35] additional stop condition for sync FORWARD mode --- .../org/aion/zero/impl/sync/PeerState.java | 62 ++++++++----- .../aion/zero/impl/sync/TaskGetHeaders.java | 91 +++++++++++-------- .../aion/zero/impl/sync/TaskImportBlocks.java | 17 +++- 3 files changed, 110 insertions(+), 60 deletions(-) diff --git a/modAionImpl/src/org/aion/zero/impl/sync/PeerState.java b/modAionImpl/src/org/aion/zero/impl/sync/PeerState.java index 36e8020e15..7d0d52fabc 100644 --- a/modAionImpl/src/org/aion/zero/impl/sync/PeerState.java +++ b/modAionImpl/src/org/aion/zero/impl/sync/PeerState.java @@ -3,42 +3,28 @@ public class PeerState { public enum Mode { - /** - * The peer is in main-chain; use normal syncing strategy. - */ + /** The peer is in main-chain; use normal syncing strategy. */ NORMAL, - /** - * The peer is in side-chain; sync backward to find the fork point. - */ + /** The peer is in side-chain; sync backward to find the fork point. */ BACKWARD, - /** - * The peer is in side-chain; sync forward to catch up. - */ + /** The peer is in side-chain; sync forward to catch up. */ FORWARD } // TODO: enforce rules on this public enum State { - /** - * The initial state. - */ + /** The initial state. */ INITIAL, - /** - * Status request, waiting for response. - */ + /** Status request, waiting for response. */ STATUS_REQUESTED, - /** - * Block headers request, waiting for response. - */ + /** Block headers request, waiting for response. */ HEADERS_REQUESTED, - /** - * Block bodies request, waiting for response. - */ + /** Block bodies request, waiting for response. */ BODIES_REQUESTED, } @@ -46,6 +32,11 @@ public enum State { private Mode mode; private long base; + // used in FORWARD mode to prevent endlessly importing EXISTing blocks + // compute how many times to go forward without importing a new block + private int repeated; + private int maxRepeats; + // The syncing status private State state; private long lastHeaderRequest; @@ -98,4 +89,33 @@ public void setLastHeaderRequest(long lastStatusRequest) { public void resetLastHeaderRequest() { this.lastHeaderRequest = 0; } + + public int getRepeated() { + return repeated; + } + + public void resetRepeated() { + this.repeated = 0; + } + + public void incRepeated() { + this.repeated++; + } + + /** + * This number is set based on the BACKWARD step size and the size of each requested batch in + * FORWARD mode. Passing the number of repeats allowed means that we have entered in the + * previous BACKWARD step. If that step would have been viable, we never would have made another + * step back, so it effectively ends the FORWARD pass. + * + * @return The number of times that a node in FORWARD mode can import only blocks that already + * EXIST. + */ + public int getMaxRepeats() { + return maxRepeats; + } + + public void setMaxRepeats(int maxRepeats) { + this.maxRepeats = maxRepeats; + } } diff --git a/modAionImpl/src/org/aion/zero/impl/sync/TaskGetHeaders.java b/modAionImpl/src/org/aion/zero/impl/sync/TaskGetHeaders.java index 7c8cf75f91..31df4901af 100644 --- a/modAionImpl/src/org/aion/zero/impl/sync/TaskGetHeaders.java +++ b/modAionImpl/src/org/aion/zero/impl/sync/TaskGetHeaders.java @@ -29,21 +29,18 @@ package org.aion.zero.impl.sync; -import org.aion.p2p.INode; -import org.aion.p2p.IP2pMgr; -import org.aion.zero.impl.sync.msg.ReqBlocksHeaders; -import org.slf4j.Logger; - import java.math.BigInteger; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Random; import java.util.stream.Collectors; +import org.aion.p2p.INode; +import org.aion.p2p.IP2pMgr; +import org.aion.zero.impl.sync.msg.ReqBlocksHeaders; +import org.slf4j.Logger; -/** - * @author chris - */ +/** @author chris */ final class TaskGetHeaders implements Runnable { private final IP2pMgr p2p; @@ -58,7 +55,12 @@ final class TaskGetHeaders implements Runnable { private final Random random = new Random(System.currentTimeMillis()); - TaskGetHeaders(IP2pMgr p2p, long selfNumber, BigInteger selfTd, Map peerStates, Logger log) { + TaskGetHeaders( + IP2pMgr p2p, + long selfNumber, + BigInteger selfTd, + Map peerStates, + Logger log) { this.p2p = p2p; this.selfNumber = selfNumber; this.selfTd = selfTd; @@ -78,8 +80,9 @@ public void run() { // higher td n.getTotalDifficulty() != null && n.getTotalDifficulty().compareTo(this.selfTd) >= 0 // not recently requested - && (now - 5000) > peerStates.computeIfAbsent(n.getIdHash(), k -> new PeerState(PeerState.Mode.NORMAL, selfNumber)).getLastHeaderRequest() - ) + && (now - 5000) > peerStates + .computeIfAbsent(n.getIdHash(), k -> new PeerState(PeerState.Mode.NORMAL, selfNumber)) + .getLastHeaderRequest()) .collect(Collectors.toList()); if (nodesFiltered.isEmpty()) { return; @@ -94,39 +97,51 @@ public void run() { // decide the start block number long from = 0; int size = 24; + + // depends on the number of blocks going BACKWARD + state.setMaxRepeats(128 / size + 1); + switch (state.getMode()) { - case NORMAL: { - // update base block - state.setBase(selfNumber); - - // normal mode - long nodeNumber = node.getBestBlockNumber(); - if (nodeNumber >= selfNumber + 128) { - from = Math.max(1, selfNumber + 1 - 4); - } else if (nodeNumber >= selfNumber - 128) { - from = Math.max(1, selfNumber + 1 - 16); - } else { - // no need to request from this node. His TD is probably corrupted. - return; + case NORMAL: + { + // update base block + state.setBase(selfNumber); + + // normal mode + long nodeNumber = node.getBestBlockNumber(); + if (nodeNumber >= selfNumber + 128) { + from = Math.max(1, selfNumber + 1 - 4); + } else if (nodeNumber >= selfNumber - 128) { + from = Math.max(1, selfNumber + 1 - 16); + } else { + // no need to request from this node. His TD is probably corrupted. + return; + } + + break; + } + case BACKWARD: + { + // step back by 128 blocks + from = Math.max(1, state.getBase() - 128); + break; + } + case FORWARD: + { + // start from base block + from = state.getBase() + 1; + break; } - - break; - } - case BACKWARD: { - // step back by 128 blocks - from = Math.max(1, state.getBase() - 128); - break; - } - case FORWARD: { - // start from base block - from = state.getBase() + 1; - break; - } } // send request if (log.isDebugEnabled()) { - log.debug("", state.getMode(), from, size, node.getIdShort()); + log.debug( + "", + state.getMode(), + from, + size, + node.getIdShort()); } ReqBlocksHeaders rbh = new ReqBlocksHeaders(from, size); this.p2p.send(node.getIdHash(), node.getIdShort(), rbh); diff --git a/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java b/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java index e69aa9ae3e..c80f292348 100644 --- a/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java +++ b/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java @@ -107,9 +107,9 @@ public void run() { "This is not supposed to happen, but the peer is sending us blocks without ask"); } + ImportResult importResult = ImportResult.IMPORTED_NOT_BEST; for (AionBlock b : batch) { long t1 = System.currentTimeMillis(); - ImportResult importResult; try { importResult = this.chain.tryToConnect(b); } catch (Throwable e) { @@ -159,6 +159,7 @@ public void run() { // we found the fork point state.setMode(PeerState.Mode.FORWARD); state.setBase(lastBlock); + state.resetRepeated(); } else if (mode == PeerState.Mode.FORWARD) { // continue state.setBase(lastBlock); @@ -174,6 +175,14 @@ public void run() { } } } + // if the maximum number of repeats is passed + // then the peer is stuck endlessly importing old blocks + // otherwise it would have found an IMPORTED block already + if (state.getRepeated() >= state.getMaxRepeats()) { + state.setMode(PeerState.Mode.NORMAL); + state.setBase(chain.getBestBlock().getNumber()); + state.resetLastHeaderRequest(); + } } break; case NO_PARENT: @@ -196,6 +205,12 @@ public void run() { } } + if (state != null && state.getMode() == PeerState.Mode.FORWARD && importResult == ImportResult.EXIST) { + // increment the repeat count every time + // we finish a batch of imports with EXIST + state.incRepeated(); + } + if (state != null) { state.resetLastHeaderRequest(); // so we can continue immediately } From 27a7a5255217d28f97b99d52a58226a129457a88 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Thu, 24 May 2018 16:37:37 -0400 Subject: [PATCH 24/35] new pruning configurations and default values --- .../src/org/aion/base/db/IPruneConfig.java | 7 ++ .../aion/zero/impl/AionBlockchainImpl.java | 2 +- modBoot/resource/config.xml | 14 ++-- modMcf/src/org/aion/mcf/config/CfgDb.java | 46 +++++++++++-- modMcf/src/org/aion/mcf/config/CfgPrune.java | 68 +++++++++++++------ .../org/aion/mcf/db/AbstractRepository.java | 32 ++++++--- .../aion/mcf/trie/JournalPruneDataSource.java | 2 +- 7 files changed, 126 insertions(+), 45 deletions(-) diff --git a/modAionBase/src/org/aion/base/db/IPruneConfig.java b/modAionBase/src/org/aion/base/db/IPruneConfig.java index 871537b85a..3b93b1fb50 100644 --- a/modAionBase/src/org/aion/base/db/IPruneConfig.java +++ b/modAionBase/src/org/aion/base/db/IPruneConfig.java @@ -14,6 +14,13 @@ public interface IPruneConfig { */ boolean isEnabled(); + /** + * Indicates if archiving should be enabled or disabled. + * + * @return {@code true} when archiving enabled, {@code false} when archiving disabled. + */ + boolean isArchived(); + /** * @return the number of topmost blocks for which the full data should be maintained on disk. */ diff --git a/modAionImpl/src/org/aion/zero/impl/AionBlockchainImpl.java b/modAionImpl/src/org/aion/zero/impl/AionBlockchainImpl.java index 650bcebcba..0fc6955479 100644 --- a/modAionImpl/src/org/aion/zero/impl/AionBlockchainImpl.java +++ b/modAionImpl/src/org/aion/zero/impl/AionBlockchainImpl.java @@ -1351,7 +1351,7 @@ public synchronized boolean recoverWorldState(IRepository repository, AionBlock } long blockNumber = block.getNumber(); - LOG.info("Corrupt world state at block hash: {}, number: {}." + LOG.info("Pruned or corrupt world state at block hash: {}, number: {}." + " Looking for ancestor block with valid world state ...", block.getShortHash(), blockNumber); AionRepositoryImpl repo = (AionRepositoryImpl) repository; diff --git a/modBoot/resource/config.xml b/modBoot/resource/config.xml index b81265be81..872285a788 100644 --- a/modBoot/resource/config.xml +++ b/modBoot/resource/config.xml @@ -64,15 +64,11 @@ database true - - - - true - - 128 - - 10000 - + + + + + FULL leveldb diff --git a/modMcf/src/org/aion/mcf/config/CfgDb.java b/modMcf/src/org/aion/mcf/config/CfgDb.java index 3ec6abc924..fb0d075099 100644 --- a/modMcf/src/org/aion/mcf/config/CfgDb.java +++ b/modMcf/src/org/aion/mcf/config/CfgDb.java @@ -62,6 +62,7 @@ public static class Names { private boolean compression; private boolean check_integrity; private CfgPrune prune; + private String prune_option; /** * Enabling expert mode allows more detailed database configurations. @@ -78,7 +79,8 @@ public CfgDb() { this.vendor = DBVendor.LEVELDB.toValue(); this.compression = false; this.check_integrity = true; - this.prune = new CfgPrune(); + this.prune = new CfgPrune(false); + this.prune_option = "full"; if (expert) { this.specificConfig = new HashMap<>(); @@ -100,8 +102,30 @@ public void fromXML(final XMLStreamReader sr) throws XMLStreamException { case "check_integrity": this.check_integrity = Boolean.parseBoolean(Cfg.readValue(sr)); break; - case "prune": - this.prune.fromXML(sr); + case "state-storage": + prune_option = Cfg.readValue(sr).toLowerCase(); + switch (prune_option) { + // journal prune only + case "top": + { + this.prune = new CfgPrune(128); + break; + } + // journal prune with archived states + case "spread": + { + this.prune = new CfgPrune(128, 10000); + break; + } + // the default is no pruning + case "full": + default: + { + this.prune = new CfgPrune(false); + this.prune_option = "full"; + break; + } + } break; // parameter considered only when expert==false case "vendor": @@ -220,8 +244,20 @@ public String toXML() { xmlWriter.writeEndElement(); xmlWriter.writeCharacters("\r\n\t\t"); - xmlWriter.writeComment("Configuration for data pruning behavior."); - this.prune.toXML(xmlWriter); + xmlWriter.writeComment( + "Data pruning behavior for the state database. Options: FULL, TOP, SPREAD."); + xmlWriter.writeCharacters("\r\n\t\t"); + xmlWriter.writeComment("FULL: the state is not pruned"); + xmlWriter.writeCharacters("\r\n\t\t"); + xmlWriter.writeComment( + "TOP: the state is kept only for the top K blocks; limits sync to branching only within the stored blocks"); + xmlWriter.writeCharacters("\r\n\t\t"); + xmlWriter.writeComment( + "SPREAD: the state is kept for the top K blocks and at regular block intervals"); + xmlWriter.writeCharacters("\r\n\t\t"); + xmlWriter.writeStartElement("state-storage"); + xmlWriter.writeCharacters(this.prune_option.toUpperCase()); + xmlWriter.writeEndElement(); if (!expert) { xmlWriter.writeCharacters("\r\n\t\t"); diff --git a/modMcf/src/org/aion/mcf/config/CfgPrune.java b/modMcf/src/org/aion/mcf/config/CfgPrune.java index 0fb56f98e9..842c483fe2 100644 --- a/modMcf/src/org/aion/mcf/config/CfgPrune.java +++ b/modMcf/src/org/aion/mcf/config/CfgPrune.java @@ -41,22 +41,36 @@ public class CfgPrune implements IPruneConfig { private boolean enabled; - private int current; - private int archive; + private boolean archived; + private int current_count = MINIMUM_CURRENT_COUNT; + private int archive_rate = MINIMUM_ARCHIVE_RATE; private static final int MINIMUM_CURRENT_COUNT = 128; private static final int MINIMUM_ARCHIVE_RATE = 1000; - public CfgPrune() { + public CfgPrune(boolean _enabled) { + this.enabled = _enabled; + this.archived = _enabled; + } + + public CfgPrune(int _current_count) { + // enable journal pruning this.enabled = true; - this.current = MINIMUM_CURRENT_COUNT; - this.archive = MINIMUM_ARCHIVE_RATE; + this.current_count = + _current_count > MINIMUM_CURRENT_COUNT ? _current_count : MINIMUM_CURRENT_COUNT; + // disable archiving + this.archived = false; } - public CfgPrune(boolean _enabled) { - this.enabled = _enabled; - this.current = MINIMUM_CURRENT_COUNT; - this.archive = MINIMUM_ARCHIVE_RATE; + public CfgPrune(int _current_count, int _archive_rate) { + // enable journal pruning + this.enabled = true; + this.current_count = + _current_count > MINIMUM_CURRENT_COUNT ? _current_count : MINIMUM_CURRENT_COUNT; + // enable archiving + this.archived = true; + this.archive_rate = + _archive_rate > MINIMUM_ARCHIVE_RATE ? _archive_rate : MINIMUM_ARCHIVE_RATE; } public void fromXML(final XMLStreamReader sr) throws XMLStreamException { @@ -70,18 +84,21 @@ public void fromXML(final XMLStreamReader sr) throws XMLStreamException { case "enabled": this.enabled = Boolean.parseBoolean(Cfg.readValue(sr)); break; + case "archived": + this.archived = Boolean.parseBoolean(Cfg.readValue(sr)); + break; case "current_count": - this.current = Integer.parseInt(Cfg.readValue(sr)); + this.current_count = Integer.parseInt(Cfg.readValue(sr)); // must be at least MINIMUM_CURRENT_COUNT - if (this.current < MINIMUM_CURRENT_COUNT) { - this.current = MINIMUM_CURRENT_COUNT; + if (this.current_count < MINIMUM_CURRENT_COUNT) { + this.current_count = MINIMUM_CURRENT_COUNT; } break; case "archive_rate": - this.archive = Integer.parseInt(Cfg.readValue(sr)); + this.archive_rate = Integer.parseInt(Cfg.readValue(sr)); // must be at least MINIMUM_ARCHIVE_RATE - if (this.archive < MINIMUM_ARCHIVE_RATE) { - this.archive = MINIMUM_ARCHIVE_RATE; + if (this.archive_rate < MINIMUM_ARCHIVE_RATE) { + this.archive_rate = MINIMUM_ARCHIVE_RATE; } break; default: @@ -106,12 +123,19 @@ public void toXML(XMLStreamWriter xmlWriter) throws XMLStreamException { xmlWriter.writeCharacters(String.valueOf(this.enabled)); xmlWriter.writeEndElement(); + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeComment("Boolean value. Enable/disable database archiving."); + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeStartElement("archived"); + xmlWriter.writeCharacters(String.valueOf(this.archived)); + xmlWriter.writeEndElement(); + xmlWriter.writeCharacters("\r\n\t\t\t"); xmlWriter.writeComment( "Integer value with minimum set to 128. Only blocks older than best block level minus this number are candidates for pruning."); xmlWriter.writeCharacters("\r\n\t\t\t"); xmlWriter.writeStartElement("current_count"); - xmlWriter.writeCharacters(String.valueOf(this.current)); + xmlWriter.writeCharacters(String.valueOf(this.current_count)); xmlWriter.writeEndElement(); xmlWriter.writeCharacters("\r\n\t\t\t"); @@ -119,24 +143,30 @@ public void toXML(XMLStreamWriter xmlWriter) throws XMLStreamException { "Integer value with minimum set to 1000. States for blocks that are exact multiples of this number will not be pruned."); xmlWriter.writeCharacters("\r\n\t\t\t"); xmlWriter.writeStartElement("archive_rate"); - xmlWriter.writeCharacters(String.valueOf(this.archive)); + xmlWriter.writeCharacters(String.valueOf(this.archive_rate)); xmlWriter.writeEndElement(); xmlWriter.writeCharacters("\r\n\t\t"); xmlWriter.writeEndElement(); } + @Override public boolean isEnabled() { return enabled; } + @Override + public boolean isArchived() { + return archived; + } + @Override public int getCurrentCount() { - return current; + return current_count; } @Override public int getArchiveRate() { - return archive; + return archive_rate; } } diff --git a/modMcf/src/org/aion/mcf/db/AbstractRepository.java b/modMcf/src/org/aion/mcf/db/AbstractRepository.java index 70c4afd00b..60efd98d76 100644 --- a/modMcf/src/org/aion/mcf/db/AbstractRepository.java +++ b/modMcf/src/org/aion/mcf/db/AbstractRepository.java @@ -194,11 +194,6 @@ protected void initializeDatabasesAndCaches() throws Exception { this.stateDatabase = connectAndOpen(sharedProps); databaseGroup.add(stateDatabase); - // using state config for state_archive - sharedProps.setProperty(Props.DB_NAME, STATE_ARCHIVE_DB); - this.stateArchiveDatabase = connectAndOpen(sharedProps); - databaseGroup.add(stateArchiveDatabase); - // getting transaction specific properties sharedProps = cfg.getDatabaseConfig(TRANSACTION_DB); sharedProps.setProperty(Props.ENABLE_LOCKING, "false"); @@ -257,22 +252,39 @@ protected void initializeDatabasesAndCaches() throws Exception { // Setup the cache for transaction data source. this.detailsDS = new DetailsDataStore<>(detailsDatabase, storageDatabase, this.cfg); - stateWithArchive = new ArchivedDataSource(stateDatabase, stateArchiveDatabase); - stateDSPrune = new JournalPruneDataSource(stateWithArchive); // pruning config pruneEnabled = this.cfg.getPruneConfig().isEnabled(); pruneBlockCount = this.cfg.getPruneConfig().getCurrentCount(); archiveRate = this.cfg.getPruneConfig().getArchiveRate(); - if (pruneEnabled) { + if (pruneEnabled && this.cfg.getPruneConfig().isArchived()) { + // using state config for state_archive + sharedProps = cfg.getDatabaseConfig(STATE_DB); + sharedProps.setProperty(Props.ENABLE_LOCKING, "false"); + sharedProps.setProperty(Props.DB_PATH, cfg.getDbPath()); + sharedProps.setProperty(Props.DB_NAME, STATE_ARCHIVE_DB); + this.stateArchiveDatabase = connectAndOpen(sharedProps); + databaseGroup.add(stateArchiveDatabase); + + stateWithArchive = new ArchivedDataSource(stateDatabase, stateArchiveDatabase); + stateDSPrune = new JournalPruneDataSource(stateWithArchive); + LOGGEN.info( - "Pruning ENABLED. Block count set to {} and archive rate set to {}.", + "Pruning and archiving ENABLED. Top block count set to {} and archive rate set to {}.", pruneBlockCount, archiveRate); } else { - stateDSPrune.setPruneEnabled(false); + stateArchiveDatabase = null; + stateWithArchive = null; + stateDSPrune = new JournalPruneDataSource(stateDatabase); + + if (pruneEnabled) { + LOGGEN.info("Pruning ENABLED. Top block count set to {}.", pruneBlockCount); + } } + + stateDSPrune.setPruneEnabled(pruneEnabled); } catch (Exception e) { // Setting up databases and caches went wrong. throw e; } diff --git a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java index 3141f79d65..25f74eeb9f 100644 --- a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java +++ b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java @@ -90,7 +90,7 @@ public String toString() { // block hash => updates private LinkedHashMap blockUpdates = new LinkedHashMap<>(); private Updates currentUpdates = new Updates(); - private AtomicBoolean enabled = new AtomicBoolean(true); + private AtomicBoolean enabled = new AtomicBoolean(false); private final boolean hasArchive; public JournalPruneDataSource(IByteArrayKeyValueStore src) { From ada9502dfa1f310d2e3b5271e47a1051d6e1db3b Mon Sep 17 00:00:00 2001 From: Alexandra Date: Fri, 25 May 2018 11:00:23 -0400 Subject: [PATCH 25/35] correcting broken test cases --- .../aion/mcf/trie/JournalPruneDataSource.java | 6 ++-- .../mcf/trie/JournalPruneDataSourceTest.java | 35 +++++++++++++++++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java index 25f74eeb9f..05aca77569 100644 --- a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java +++ b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java @@ -151,10 +151,11 @@ public void put(byte[] key, byte[] value) { } public void delete(byte[] key) { + checkNotNull(key); if (!enabled.get()) { + check(); return; } - checkNotNull(key); lock.writeLock().lock(); @@ -376,10 +377,11 @@ public void commitBatch() { @Override public void deleteBatch(Collection keys) { + checkNotNull(keys); if (!enabled.get()) { + check(); return; } - checkNotNull(keys); lock.writeLock().lock(); diff --git a/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java b/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java index 6e0dab3af0..a32657474b 100644 --- a/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java +++ b/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java @@ -475,6 +475,8 @@ public void testDelete_wPrune() { @Test public void testDeleteBatch_wPrune() { + db.setPruneEnabled(true); + // ensure existence Map map = new HashMap<>(); map.put(k1, v1); @@ -612,6 +614,7 @@ public void testIsEmpty_wClosedDatabase() { @Test(expected = RuntimeException.class) public void testIsEmpty_wClosedDatabase_wInsertedKeys() { + db.setPruneEnabled(true); db.put(randomBytes(32), randomBytes(32)); source_db.close(); @@ -659,7 +662,18 @@ public void testPut_wClosedDatabase_wNullValue() { } @Test(expected = RuntimeException.class) - public void testDelete_wClosedDatabase() { + public void testDelete_wClosedDatabase_wPrune() { + db.setPruneEnabled(true); + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + // attempt delete on closed db + db.delete(randomBytes(32)); + } + + @Test(expected = RuntimeException.class) + public void testDelete_wClosedDatabase_woPrune() { + db.setPruneEnabled(false); source_db.close(); assertThat(source_db.isOpen()).isFalse(); @@ -682,7 +696,23 @@ public void testPutBatch_wClosedDatabase() { } @Test(expected = RuntimeException.class) - public void testDeleteBatch_wClosedDatabase() { + public void testDeleteBatch_wClosedDatabase_wPrune() { + db.setPruneEnabled(true); + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + List list = new ArrayList<>(); + list.add(randomBytes(32)); + list.add(randomBytes(32)); + list.add(randomBytes(32)); + + // attempt deleteBatch on closed db + db.deleteBatch(list); + } + + @Test(expected = RuntimeException.class) + public void testDeleteBatch_wClosedDatabase_woPrune() { + db.setPruneEnabled(false); source_db.close(); assertThat(source_db.isOpen()).isFalse(); @@ -1297,6 +1327,7 @@ public void pruningTest_wBatch() { @Test public void pruningTest_woStoredLevel() { + db.setPruneEnabled(true); source_db.put(k2, v2); source_db.put(k3, v3); From 0e4880adce59358ef16a0444819c2408b2da241c Mon Sep 17 00:00:00 2001 From: Alexandra Date: Fri, 25 May 2018 15:53:41 -0400 Subject: [PATCH 26/35] extracting the buildGenesis functionality to a utils class --- .../src/org/aion/zero/impl/AionHub.java | 34 ++-------- .../src/org/aion/zero/impl/AionHubUtils.java | 65 +++++++++++++++++++ 2 files changed, 72 insertions(+), 27 deletions(-) create mode 100644 modAionImpl/src/org/aion/zero/impl/AionHubUtils.java diff --git a/modAionImpl/src/org/aion/zero/impl/AionHub.java b/modAionImpl/src/org/aion/zero/impl/AionHub.java index 88fdf11fa6..f706adbcd7 100644 --- a/modAionImpl/src/org/aion/zero/impl/AionHub.java +++ b/modAionImpl/src/org/aion/zero/impl/AionHub.java @@ -282,9 +282,13 @@ private void loadBlockchain() { "Corrupt world state for genesis block hash: " + genesis.getShortHash() + ", number: " + genesis .getNumber() + "."); - buildGenesis(genesis); + AionHubUtils.buildGenesis(genesis, repository); - LOG.info("Rebuilding genesis block SUCCEEDED."); + if (repository.isValidRoot(genesis.getStateRoot())) { + LOG.info("Rebuilding genesis block SUCCEEDED."); + } else { + LOG.info("Rebuilding genesis block FAILED."); + } } recovered = this.blockchain.recoverWorldState(this.repository, bestBlock); @@ -335,7 +339,7 @@ private void loadBlockchain() { AionGenesis genesis = cfg.getGenesis(); - buildGenesis(genesis); + AionHubUtils.buildGenesis(genesis, repository); blockchain.setBestBlock(genesis); blockchain.setTotalDifficulty(genesis.getCumulativeDifficulty()); @@ -391,30 +395,6 @@ private void loadBlockchain() { // this.repository.getBlockStore().load(); } - private void buildGenesis(AionGenesis genesis) { - // initialization section for network balance contract - IRepositoryCache track = repository.startTracking(); - - Address networkBalanceAddress = PrecompiledContracts.totalCurrencyAddress; - track.createAccount(networkBalanceAddress); - - for (Map.Entry addr : genesis.getNetworkBalances().entrySet()) { - track.addStorageRow( - networkBalanceAddress, - new DataWord(addr.getKey()), - new DataWord(addr.getValue())); - } - - for (Address addr : genesis.getPremine().keySet()) { - track.createAccount(addr); - track.addBalance(addr, genesis.getPremine().get(addr).getBalance()); - } - track.flush(); - - this.repository.commitBlock(genesis.getHeader()); - this.repository.getBlockStore().saveBlock(genesis, genesis.getCumulativeDifficulty(), true); - } - public void close() { LOG.info(""); diff --git a/modAionImpl/src/org/aion/zero/impl/AionHubUtils.java b/modAionImpl/src/org/aion/zero/impl/AionHubUtils.java new file mode 100644 index 0000000000..1beab75533 --- /dev/null +++ b/modAionImpl/src/org/aion/zero/impl/AionHubUtils.java @@ -0,0 +1,65 @@ +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: + * Aion foundation. + ******************************************************************************/ +package org.aion.zero.impl; + +import java.math.BigInteger; +import java.util.Map; +import org.aion.base.db.IRepositoryCache; +import org.aion.base.type.Address; +import org.aion.mcf.vm.types.DataWord; +import org.aion.vm.PrecompiledContracts; +import org.aion.zero.impl.db.AionRepositoryImpl; + +/** {@link AionHub} functionality where a full instantiation of the class is not desirable. */ +public class AionHubUtils { + + public static void buildGenesis(AionGenesis genesis, AionRepositoryImpl repository) { + // initialization section for network balance contract + IRepositoryCache track = repository.startTracking(); + + Address networkBalanceAddress = PrecompiledContracts.totalCurrencyAddress; + track.createAccount(networkBalanceAddress); + + for (Map.Entry addr : genesis.getNetworkBalances().entrySet()) { + track.addStorageRow( + networkBalanceAddress, + new DataWord(addr.getKey()), + new DataWord(addr.getValue())); + } + + for (Address addr : genesis.getPremine().keySet()) { + track.createAccount(addr); + track.addBalance(addr, genesis.getPremine().get(addr).getBalance()); + } + track.flush(); + + repository.commitBlock(genesis.getHeader()); + repository.getBlockStore().saveBlock(genesis, genesis.getCumulativeDifficulty(), true); + } +} From 188a483e9092cc94cb4ab7b95914d95b452aa5ec Mon Sep 17 00:00:00 2001 From: Alexandra Date: Fri, 25 May 2018 16:59:22 -0400 Subject: [PATCH 27/35] state pruning as CLI options --- .../src/org/aion/zero/impl/cli/Cli.java | 14 ++++ .../aion/zero/impl/db/AionRepositoryImpl.java | 4 + .../org/aion/zero/impl/db/RecoveryUtils.java | 49 ++++++++++++ modMcf/src/org/aion/mcf/config/CfgDb.java | 80 +++++++++++++------ 4 files changed, 121 insertions(+), 26 deletions(-) diff --git a/modAionImpl/src/org/aion/zero/impl/cli/Cli.java b/modAionImpl/src/org/aion/zero/impl/cli/Cli.java index 685ecb84e1..735a950c1f 100644 --- a/modAionImpl/src/org/aion/zero/impl/cli/Cli.java +++ b/modAionImpl/src/org/aion/zero/impl/cli/Cli.java @@ -129,6 +129,20 @@ public int call(final String[] args, final Cfg cfg) { } } break; + case "--state": { + String pruning_type = "full"; + if (args.length >= 2) { + pruning_type = args[1]; + } + try { + RecoveryUtils.pruneOrRecoverState(pruning_type); + } catch (Throwable t) { + System.out.println("Reorganizing the state storage FAILED due to:"); + t.printStackTrace(); + return 1; + } + break; + } case "--dump-state-size": long block_count = 2L; diff --git a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java index f047665c3a..177070800e 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java +++ b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java @@ -698,6 +698,10 @@ public IByteArrayKeyValueDatabase getStateDatabase() { return this.stateDatabase; } + public IByteArrayKeyValueDatabase getStateArchiveDatabase() { + return this.stateArchiveDatabase; + } + /** * Retrieves the underlying details database that sits below all caches. This is usually * provided by {@link org.aion.db.impl.mockdb.MockDB} or {@link org.aion.db.impl.mockdb.MockDB}. diff --git a/modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java b/modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java index 7afe26f58c..fba8844118 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java +++ b/modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java @@ -27,8 +27,11 @@ import java.util.Map; import org.aion.base.type.IBlock; import org.aion.log.AionLoggerFactory; +import org.aion.mcf.config.CfgDb; import org.aion.mcf.db.IBlockStoreBase; import org.aion.zero.impl.AionBlockchainImpl; +import org.aion.zero.impl.AionGenesis; +import org.aion.zero.impl.AionHubUtils; import org.aion.zero.impl.config.CfgAion; import org.aion.zero.impl.core.IAionBlockchain; import org.aion.zero.impl.types.AionBlock; @@ -324,4 +327,50 @@ public static void printStateTrieDump(long blockNumber) { repository.close(); } + + public static void pruneOrRecoverState(String pruning_type) { + // ensure mining is disabled + CfgAion cfg = CfgAion.inst(); + cfg.fromXML(); + cfg.getConsensus().setMining(false); + + // setting pruning to the version requested + CfgDb.PruneOption option = CfgDb.PruneOption.fromValue(pruning_type); + cfg.getDb().setPrune(option.toString()); + + System.out.println("Reorganizing the state storage to " + option + " mode ..."); + + Map cfgLog = new HashMap<>(); + cfgLog.put("DB", "ERROR"); + cfgLog.put("CONS", "ERROR"); + + AionLoggerFactory.init(cfgLog); + + AionBlockchainImpl chain = AionBlockchainImpl.inst(); + AionRepositoryImpl repo = (AionRepositoryImpl) chain.getRepository(); + AionBlockStore store = repo.getBlockStore(); + + // dropping old state database + System.out.println("Deleting old data ..."); + repo.getStateDatabase().drop(); + if (pruning_type.equals("spread")) { + repo.getStateArchiveDatabase().drop(); + } + + // recover genesis + System.out.println("Rebuilding genesis block ..."); + AionGenesis genesis = cfg.getGenesis(); + AionHubUtils.buildGenesis(genesis, repo); + + // recover all blocks + AionBlock block = store.getBestBlock(); + System.out.println( + "Rebuilding the main chain " + + block.getNumber() + + " blocks (may take a while) ..."); + chain.recoverWorldState(repo, block); + + repo.close(); + System.out.println("Reorganizing the state storage COMPLETE."); + } } diff --git a/modMcf/src/org/aion/mcf/config/CfgDb.java b/modMcf/src/org/aion/mcf/config/CfgDb.java index fb0d075099..5a29267585 100644 --- a/modMcf/src/org/aion/mcf/config/CfgDb.java +++ b/modMcf/src/org/aion/mcf/config/CfgDb.java @@ -62,7 +62,7 @@ public static class Names { private boolean compression; private boolean check_integrity; private CfgPrune prune; - private String prune_option; + private PruneOption prune_option; /** * Enabling expert mode allows more detailed database configurations. @@ -80,7 +80,7 @@ public CfgDb() { this.compression = false; this.check_integrity = true; this.prune = new CfgPrune(false); - this.prune_option = "full"; + this.prune_option = PruneOption.FULL; if (expert) { this.specificConfig = new HashMap<>(); @@ -103,29 +103,7 @@ public void fromXML(final XMLStreamReader sr) throws XMLStreamException { this.check_integrity = Boolean.parseBoolean(Cfg.readValue(sr)); break; case "state-storage": - prune_option = Cfg.readValue(sr).toLowerCase(); - switch (prune_option) { - // journal prune only - case "top": - { - this.prune = new CfgPrune(128); - break; - } - // journal prune with archived states - case "spread": - { - this.prune = new CfgPrune(128, 10000); - break; - } - // the default is no pruning - case "full": - default: - { - this.prune = new CfgPrune(false); - this.prune_option = "full"; - break; - } - } + setPrune(Cfg.readValue(sr)); break; // parameter considered only when expert==false case "vendor": @@ -256,7 +234,7 @@ public String toXML() { "SPREAD: the state is kept for the top K blocks and at regular block intervals"); xmlWriter.writeCharacters("\r\n\t\t"); xmlWriter.writeStartElement("state-storage"); - xmlWriter.writeCharacters(this.prune_option.toUpperCase()); + xmlWriter.writeCharacters(this.prune_option.toString()); xmlWriter.writeEndElement(); if (!expert) { @@ -306,6 +284,56 @@ public CfgPrune getPrune() { return this.prune; } + public enum PruneOption { + FULL, + TOP, + SPREAD; + + @Override + public String toString() { + return this.name(); + } + + public static PruneOption fromValue(String value) { + value = value.toUpperCase(); + + if (value != null) { + for (PruneOption color : values()) { + if (color.toString().equals(value)) { + return color; + } + } + } + + // return default value + return getDefault(); + } + + public static PruneOption getDefault() { + return FULL; + } + } + + public void setPrune(String _prune_option) { + this.prune_option = PruneOption.fromValue(_prune_option); + + switch (prune_option) { + case TOP: + // journal prune only + this.prune = new CfgPrune(128); + break; + case SPREAD: + // journal prune with archived states + this.prune = new CfgPrune(128, 10000); + break; + case FULL: + default: + // the default is no pruning + this.prune = new CfgPrune(false); + break; + } + } + public Map asProperties() { Map propSet = new HashMap<>(); From d87ac87e3b71d48be481fbd05638ffe3d54b5470 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Fri, 25 May 2018 17:25:56 -0400 Subject: [PATCH 28/35] skipping full batch when in FORWARD mode and last block satisfies requirement --- .../aion/zero/impl/sync/TaskImportBlocks.java | 74 ++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java b/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java index c80f292348..75bda51092 100644 --- a/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java +++ b/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java @@ -108,6 +108,76 @@ public void run() { } ImportResult importResult = ImportResult.IMPORTED_NOT_BEST; + + // importing last block in batch to see if we can skip batch + if (state != null && state.getMode() == PeerState.Mode.FORWARD && batch.size() > 0) { + AionBlock b = batch.get(batch.size() - 1); + + long t1 = System.currentTimeMillis(); + try { + importResult = this.chain.tryToConnect(b); + } catch (Throwable e) { + log.error(" {}", e.toString()); + if (e.getMessage() != null + && e.getMessage().contains("No space left on device")) { + log.error("Shutdown due to lack of disk space."); + System.exit(0); + } + continue; + } + long t2 = System.currentTimeMillis(); + log.info( + "", + bw.getDisplayId(), + state.getMode(), + b.getShortHash(), + b.getNumber(), + b.getTransactionsList().size(), + importResult, + t2 - t1); + + switch (importResult) { + case IMPORTED_BEST: + case IMPORTED_NOT_BEST: + case EXIST: + { + importedBlockHashes.put(ByteArrayWrapper.wrap(b.getHash()), true); + + long lastBlock = batch.get(batch.size() - 1).getNumber(); + + // continue + state.setBase(lastBlock); + // if the imported best block, switch back to normal mode + if (importResult == ImportResult.IMPORTED_BEST) { + state.setMode(PeerState.Mode.NORMAL); + // switch peers to NORMAL otherwise they may never switch back + for (PeerState peerState : peerStates.values()) { + if (peerState.getMode() != PeerState.Mode.NORMAL) { + peerState.setMode(PeerState.Mode.NORMAL); + peerState.setBase(b.getNumber()); + peerState.resetLastHeaderRequest(); + } + } + } + // if the maximum number of repeats is passed + // then the peer is stuck endlessly importing old blocks + // otherwise it would have found an IMPORTED block already + if (state.getRepeated() >= state.getMaxRepeats()) { + state.setMode(PeerState.Mode.NORMAL); + state.setBase(chain.getBestBlock().getNumber()); + state.resetLastHeaderRequest(); + } + + // since last import worked skipping the batch + batch.clear(); + log.info("Forward skip."); + break; + } + default: + break; + } + } + for (AionBlock b : batch) { long t1 = System.currentTimeMillis(); try { @@ -205,7 +275,9 @@ public void run() { } } - if (state != null && state.getMode() == PeerState.Mode.FORWARD && importResult == ImportResult.EXIST) { + if (state != null + && state.getMode() == PeerState.Mode.FORWARD + && importResult == ImportResult.EXIST) { // increment the repeat count every time // we finish a batch of imports with EXIST state.incRepeated(); From d415e4ac897fd78148d0b9d35c69a098be6e56a2 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Fri, 25 May 2018 18:12:34 -0400 Subject: [PATCH 29/35] checking for prune restrictions before importing blocks --- .../aion/zero/impl/AionBlockchainImpl.java | 33 ++++++++++++------- .../aion/zero/impl/db/AionRepositoryImpl.java | 17 ++++++---- .../aion/zero/impl/sync/TaskImportBlocks.java | 13 ++++++++ 3 files changed, 45 insertions(+), 18 deletions(-) diff --git a/modAionImpl/src/org/aion/zero/impl/AionBlockchainImpl.java b/modAionImpl/src/org/aion/zero/impl/AionBlockchainImpl.java index 0fc6955479..f7ff7c7533 100644 --- a/modAionImpl/src/org/aion/zero/impl/AionBlockchainImpl.java +++ b/modAionImpl/src/org/aion/zero/impl/AionBlockchainImpl.java @@ -122,7 +122,7 @@ public class AionBlockchainImpl implements IAionBlockchain { private A0BCConfig config; private long exitOn = Long.MAX_VALUE; - private IRepository repository; + private AionRepositoryImpl repository; private IRepositoryCache track; private TransactionStore transactionStore; private AionBlock bestBlock; @@ -221,7 +221,7 @@ private AionBlockchainImpl() { } protected AionBlockchainImpl(final A0BCConfig config, - final IRepository repository, + final AionRepositoryImpl repository, final ChainConfiguration chainConfig) { this.config = config; this.repository = repository; @@ -237,7 +237,7 @@ protected AionBlockchainImpl(final A0BCConfig config, this.parentHeaderValidator = this.chainConfiguration.createParentHeaderValidator(); this.blockHeaderValidator = this.chainConfiguration.createBlockHeaderValidator(); - this.transactionStore = ((AionRepositoryImpl) this.repository).getTransactionStore(); + this.transactionStore = this.repository.getTransactionStore(); this.minerCoinbase = this.config.getMinerCoinbase(); @@ -283,7 +283,7 @@ void setEventManager(IEventMgr eventManager) { } public AionBlockStore getBlockStore() { - return (AionBlockStore) repository.getBlockStore(); + return repository.getBlockStore(); } /** @@ -422,7 +422,7 @@ private static byte[] calcTxTrie(List transactions) { return txsState.getRootHash(); } - public IRepository getRepository() { + public AionRepositoryImpl getRepository() { return repository; } @@ -430,7 +430,7 @@ private State pushState(byte[] bestBlockHash) { State push = stateStack.push(new State()); this.bestBlock = getBlockStore().getBlockByHash(bestBlockHash); this.totalDifficulty = getBlockStore().getTotalDifficultyForHash(bestBlockHash); - this.repository = this.repository.getSnapshotTo(this.bestBlock.getStateRoot()); + this.repository = (AionRepositoryImpl) this.repository.getSnapshotTo(this.bestBlock.getStateRoot()); return push; } @@ -511,6 +511,17 @@ public boolean skipTryToConnect(long blockNumber) { return blockNumber > current + 32 || blockNumber < current - 32; } + /** + * Heuristic for skipping the call to tryToConnect with block number that was already pruned. + */ + public boolean isPruneRestricted(long blockNumber) { + // no restriction when not in TOP pruning mode + if (!repository.usesTopPruning()) { + return false; + } + return blockNumber < bestBlockNumber.get() - repository.getPruneBlockCount() + 1; + } + public synchronized ImportResult tryToConnect(final AionBlock block) { return tryToConnectInternal(block, System.currentTimeMillis() / THOUSAND_MS); } @@ -758,7 +769,7 @@ public synchronized AionBlockSummary add(AionBlock block, boolean rebuild) { track.rollback(); // block is bad so 'rollback' the state root to the original // state - ((AionRepositoryImpl) repository).setRoot(origRoot); + repository.setRoot(origRoot); } } @@ -771,7 +782,7 @@ public synchronized AionBlockSummary add(AionBlock block, boolean rebuild) { } transactionStore.flushBatch(); - ((AionRepositoryImpl) repository).commitBlock(block.getHeader()); + repository.commitBlock(block.getHeader()); if (LOG.isDebugEnabled()) LOG.debug("Block rebuilt: number: {}, hash: {}, TD: {}", block.getNumber(), block.getShortHash(), @@ -1066,7 +1077,7 @@ public synchronized void storeBlock(AionBlock block, List receipt } transactionStore.flushBatch(); - ((AionRepositoryImpl) repository).commitBlock(block.getHeader()); + repository.commitBlock(block.getHeader()); if (LOG.isDebugEnabled()) LOG.debug("Block saved: number: {}, hash: {}, TD: {}", block.getNumber(), block.getShortHash(), @@ -1131,7 +1142,7 @@ public void setTotalDifficulty(BigInteger totalDifficulty) { this.totalDifficulty = totalDifficulty; } - public void setRepository(IRepository repository) { + public void setRepository(AionRepositoryImpl repository) { this.repository = repository; } @@ -1320,7 +1331,7 @@ public List getListOfBodiesByHashes(List hashes) { private class State { - IRepository savedRepo = repository; + AionRepositoryImpl savedRepo = repository; AionBlock savedBest = bestBlock; BigInteger savedTD = totalDifficulty; } diff --git a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java index 177070800e..61ad71f1d8 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java +++ b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java @@ -485,13 +485,8 @@ public void setRoot(byte[] root) { } } - public void setPruneBlockCount(long pruneBlockCount) { - rwLock.writeLock().lock(); - try { - this.pruneBlockCount = pruneBlockCount; - } finally { - rwLock.writeLock().unlock(); - } + public long getPruneBlockCount() { + return this.pruneBlockCount; } public void commitBlock(A0BlockHeader blockHeader) { @@ -534,6 +529,14 @@ private void pruneBlocks(A0BlockHeader curBlock) { bestBlockNumber = curBlock.getNumber(); } + /** + * @return {@code true} when pruning is enabled and archiving is disabled, {@code false} + * otherwise + */ + public boolean usesTopPruning() { + return pruneEnabled && !stateDSPrune.isArchiveEnabled(); + } + public Trie getWorldState() { return worldState; } diff --git a/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java b/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java index 75bda51092..83d365a8e5 100644 --- a/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java +++ b/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java @@ -107,6 +107,19 @@ public void run() { "This is not supposed to happen, but the peer is sending us blocks without ask"); } + // checking if there are restrictions due to pruning + if (batch.size() > 0 && chain.isPruneRestricted(batch.get(0).getNumber())) { + // reset status if possible + if (state != null) { + state.setMode(PeerState.Mode.NORMAL); + state.setBase(chain.getBestBlock().getNumber()); + state.resetLastHeaderRequest(); + } + // will not import this batch because it will fail due to TOP mode pruning + batch.clear(); + continue; + } + ImportResult importResult = ImportResult.IMPORTED_NOT_BEST; // importing last block in batch to see if we can skip batch From eeb862748009943fa430b2aeecd8e84103280970 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Mon, 28 May 2018 11:50:42 -0400 Subject: [PATCH 30/35] moved the prune restriction to block filtering --- .../org/aion/zero/impl/sync/TaskImportBlocks.java | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java b/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java index 83d365a8e5..75973c2274 100644 --- a/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java +++ b/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java @@ -99,6 +99,7 @@ public void run() { List batch = bw.getBlocks().stream() .filter(b -> importedBlockHashes.get(ByteArrayWrapper.wrap(b.getHash())) == null) + .filter(b -> chain.isPruneRestricted(b.getNumber()) == false) .collect(Collectors.toList()); PeerState state = peerStates.get(bw.getNodeIdHash()); @@ -107,19 +108,6 @@ public void run() { "This is not supposed to happen, but the peer is sending us blocks without ask"); } - // checking if there are restrictions due to pruning - if (batch.size() > 0 && chain.isPruneRestricted(batch.get(0).getNumber())) { - // reset status if possible - if (state != null) { - state.setMode(PeerState.Mode.NORMAL); - state.setBase(chain.getBestBlock().getNumber()); - state.resetLastHeaderRequest(); - } - // will not import this batch because it will fail due to TOP mode pruning - batch.clear(); - continue; - } - ImportResult importResult = ImportResult.IMPORTED_NOT_BEST; // importing last block in batch to see if we can skip batch From 8a1160be9aef78726ce7580f3aa2ec4f848673d4 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Mon, 28 May 2018 14:22:29 -0400 Subject: [PATCH 31/35] bugfix: checking for null keys, values and batches --- .../org/aion/db/generic/TimedDatabase.java | 72 ++++++++++++++----- 1 file changed, 56 insertions(+), 16 deletions(-) diff --git a/modDbImpl/src/org/aion/db/generic/TimedDatabase.java b/modDbImpl/src/org/aion/db/generic/TimedDatabase.java index aa0fcfe372..b101f6f697 100644 --- a/modDbImpl/src/org/aion/db/generic/TimedDatabase.java +++ b/modDbImpl/src/org/aion/db/generic/TimedDatabase.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -28,17 +28,16 @@ ******************************************************************************/ package org.aion.db.generic; +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.base.util.Hex; import org.aion.log.AionLoggerFactory; import org.aion.log.LogEnum; import org.slf4j.Logger; -import java.util.Collection; -import java.util.Map; -import java.util.Optional; -import java.util.Set; - /** * Times different database operations and logs the time. * @@ -48,6 +47,7 @@ public class TimedDatabase implements IByteArrayKeyValueDatabase { /** Unlocked database. */ protected final IByteArrayKeyValueDatabase database; + protected static final Logger LOG = AionLoggerFactory.getLogger(LogEnum.DB.name()); public TimedDatabase(IByteArrayKeyValueDatabase _database) { @@ -59,7 +59,8 @@ public String toString() { return this.getClass().getSimpleName() + " over " + database.toString(); } - // IDatabase functionality ----------------------------------------------------------------------------------------- + // IDatabase functionality + // ----------------------------------------------------------------------------------------- @Override public boolean open() { @@ -164,7 +165,8 @@ public long approximateSize() { return result; } - // IKeyValueStore functionality ------------------------------------------------------------------------------------ + // IKeyValueStore functionality + // ------------------------------------------------------------------------------------ @Override public boolean isEmpty() { @@ -192,7 +194,13 @@ public Optional get(byte[] key) { Optional value = database.get(key); long t2 = System.nanoTime(); - LOG.debug(database.toString() + " get(key) in " + (t2 - t1) + " ns." + "\n\t\t\t\t\tkey = " + Hex.toHexString(key)); + LOG.debug( + database.toString() + + " get(key) in " + + (t2 - t1) + + " ns." + + "\n\t\t\t\t\tkey = " + + (key != null ? Hex.toHexString(key) : "null")); return value; } @@ -202,8 +210,15 @@ public void put(byte[] key, byte[] value) { database.put(key, value); long t2 = System.nanoTime(); - LOG.debug(database.toString() + " put(key,value) in " + (t2 - t1) + " ns." + "\n\t\t\t\t\tkey = " + Hex.toHexString(key) - + "\n\t\t\t\t\tvalue = " + Hex.toHexString(value)); + LOG.debug( + database.toString() + + " put(key,value) in " + + (t2 - t1) + + " ns." + + "\n\t\t\t\t\tkey = " + + (key != null ? Hex.toHexString(key) : "null") + + "\n\t\t\t\t\tvalue = " + + (value != null ? Hex.toHexString(value) : "null")); } @Override @@ -212,7 +227,13 @@ public void delete(byte[] key) { database.delete(key); long t2 = System.nanoTime(); - LOG.debug(database.toString() + " delete(key) in " + (t2 - t1) + " ns." + "\n\t\t\t\t\tkey = " + Hex.toHexString(key)); + LOG.debug( + database.toString() + + " delete(key) in " + + (t2 - t1) + + " ns." + + "\n\t\t\t\t\tkey = " + + (key != null ? Hex.toHexString(key) : "null")); } @Override @@ -221,7 +242,13 @@ public void putBatch(Map keyValuePairs) { database.putBatch(keyValuePairs); long t2 = System.nanoTime(); - LOG.debug(database.toString() + " putBatch(" + keyValuePairs.size() + ") in " + (t2 - t1) + " ns."); + LOG.debug( + database.toString() + + " putBatch(" + + (keyValuePairs != null ? keyValuePairs.size() : "null") + + ") in " + + (t2 - t1) + + " ns."); } @Override @@ -230,8 +257,15 @@ public void putToBatch(byte[] key, byte[] value) { database.putToBatch(key, value); long t2 = System.nanoTime(); - LOG.debug(database.toString() + " putToBatch(key,value) in " + (t2 - t1) + " ns." + "\n\t\t\t\t\tkey = " + Hex - .toHexString(key) + "\n\t\t\t\t\tvalue = " + Hex.toHexString(value)); + LOG.debug( + database.toString() + + " putToBatch(key,value) in " + + (t2 - t1) + + " ns." + + "\n\t\t\t\t\tkey = " + + Hex.toHexString(key) + + "\n\t\t\t\t\tvalue = " + + (value != null ? Hex.toHexString(value) : "null")); } @Override @@ -249,7 +283,13 @@ public void deleteBatch(Collection keys) { database.deleteBatch(keys); long t2 = System.nanoTime(); - LOG.debug(database.toString() + " deleteBatch(" + keys.size() + ") in " + (t2 - t1) + " ns."); + LOG.debug( + database.toString() + + " deleteBatch(" + + (keys != null ? keys.size() : "null") + + ") in " + + (t2 - t1) + + " ns."); } @Override From 51edfc468f1fb1e94bdf74141ead91078b8f16f4 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Tue, 29 May 2018 14:51:52 -0400 Subject: [PATCH 32/35] backward sync step as constant --- .../src/org/aion/zero/impl/sync/TaskGetHeaders.java | 10 ++++++---- modP2p/src/org/aion/p2p/P2pConstant.java | 4 +++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/modAionImpl/src/org/aion/zero/impl/sync/TaskGetHeaders.java b/modAionImpl/src/org/aion/zero/impl/sync/TaskGetHeaders.java index 31df4901af..357bb20649 100644 --- a/modAionImpl/src/org/aion/zero/impl/sync/TaskGetHeaders.java +++ b/modAionImpl/src/org/aion/zero/impl/sync/TaskGetHeaders.java @@ -40,6 +40,8 @@ import org.aion.zero.impl.sync.msg.ReqBlocksHeaders; import org.slf4j.Logger; +import static org.aion.p2p.P2pConstant.BACKWARD_SYNC_STEP; + /** @author chris */ final class TaskGetHeaders implements Runnable { @@ -99,7 +101,7 @@ public void run() { int size = 24; // depends on the number of blocks going BACKWARD - state.setMaxRepeats(128 / size + 1); + state.setMaxRepeats(BACKWARD_SYNC_STEP / size + 1); switch (state.getMode()) { case NORMAL: @@ -109,9 +111,9 @@ public void run() { // normal mode long nodeNumber = node.getBestBlockNumber(); - if (nodeNumber >= selfNumber + 128) { + if (nodeNumber >= selfNumber + BACKWARD_SYNC_STEP) { from = Math.max(1, selfNumber + 1 - 4); - } else if (nodeNumber >= selfNumber - 128) { + } else if (nodeNumber >= selfNumber - BACKWARD_SYNC_STEP) { from = Math.max(1, selfNumber + 1 - 16); } else { // no need to request from this node. His TD is probably corrupted. @@ -123,7 +125,7 @@ public void run() { case BACKWARD: { // step back by 128 blocks - from = Math.max(1, state.getBase() - 128); + from = Math.max(1, state.getBase() - BACKWARD_SYNC_STEP); break; } case FORWARD: diff --git a/modP2p/src/org/aion/p2p/P2pConstant.java b/modP2p/src/org/aion/p2p/P2pConstant.java index 00fa58e911..1584857f9c 100644 --- a/modP2p/src/org/aion/p2p/P2pConstant.java +++ b/modP2p/src/org/aion/p2p/P2pConstant.java @@ -23,5 +23,7 @@ public class P2pConstant { READ_MAX_RATE_TXBC = 20, // write queue timeout - WRITE_MSG_TIMEOUT = 5000; + WRITE_MSG_TIMEOUT = 5000, + + BACKWARD_SYNC_STEP = 128; } From 9e2d7895fd90e2bd6ce24ad383c0de395e14723d Mon Sep 17 00:00:00 2001 From: Alexandra Date: Tue, 29 May 2018 14:55:21 -0400 Subject: [PATCH 33/35] extracting repeated code into methods in TaskImportBlocks --- .../aion/zero/impl/sync/TaskImportBlocks.java | 124 +++++++----------- 1 file changed, 50 insertions(+), 74 deletions(-) diff --git a/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java b/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java index 75973c2274..3e579f5288 100644 --- a/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java +++ b/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java @@ -99,7 +99,7 @@ public void run() { List batch = bw.getBlocks().stream() .filter(b -> importedBlockHashes.get(ByteArrayWrapper.wrap(b.getHash())) == null) - .filter(b -> chain.isPruneRestricted(b.getNumber()) == false) + .filter(b -> !chain.isPruneRestricted(b.getNumber())) .collect(Collectors.toList()); PeerState state = peerStates.get(bw.getNodeIdHash()); @@ -111,31 +111,19 @@ public void run() { ImportResult importResult = ImportResult.IMPORTED_NOT_BEST; // importing last block in batch to see if we can skip batch - if (state != null && state.getMode() == PeerState.Mode.FORWARD && batch.size() > 0) { + if (state != null && state.getMode() == PeerState.Mode.FORWARD && !batch.isEmpty()) { AionBlock b = batch.get(batch.size() - 1); - long t1 = System.currentTimeMillis(); try { - importResult = this.chain.tryToConnect(b); + importResult = importBlock(b, bw.getDisplayId(), state); } catch (Throwable e) { log.error(" {}", e.toString()); - if (e.getMessage() != null - && e.getMessage().contains("No space left on device")) { + if (e.getMessage() != null && e.getMessage().contains("No space left on device")) { log.error("Shutdown due to lack of disk space."); System.exit(0); } continue; } - long t2 = System.currentTimeMillis(); - log.info( - "", - bw.getDisplayId(), - state.getMode(), - b.getShortHash(), - b.getNumber(), - b.getTransactionsList().size(), - importResult, - t2 - t1); switch (importResult) { case IMPORTED_BEST: @@ -146,28 +134,7 @@ public void run() { long lastBlock = batch.get(batch.size() - 1).getNumber(); - // continue - state.setBase(lastBlock); - // if the imported best block, switch back to normal mode - if (importResult == ImportResult.IMPORTED_BEST) { - state.setMode(PeerState.Mode.NORMAL); - // switch peers to NORMAL otherwise they may never switch back - for (PeerState peerState : peerStates.values()) { - if (peerState.getMode() != PeerState.Mode.NORMAL) { - peerState.setMode(PeerState.Mode.NORMAL); - peerState.setBase(b.getNumber()); - peerState.resetLastHeaderRequest(); - } - } - } - // if the maximum number of repeats is passed - // then the peer is stuck endlessly importing old blocks - // otherwise it would have found an IMPORTED block already - if (state.getRepeated() >= state.getMaxRepeats()) { - state.setMode(PeerState.Mode.NORMAL); - state.setBase(chain.getBestBlock().getNumber()); - state.resetLastHeaderRequest(); - } + forwardModeUpdate(state, lastBlock, importResult, b.getNumber()); // since last import worked skipping the batch batch.clear(); @@ -180,28 +147,16 @@ public void run() { } for (AionBlock b : batch) { - long t1 = System.currentTimeMillis(); try { - importResult = this.chain.tryToConnect(b); + importResult = importBlock(b, bw.getDisplayId(), state); } catch (Throwable e) { log.error(" {}", e.toString()); - if (e.getMessage() != null - && e.getMessage().contains("No space left on device")) { + if (e.getMessage() != null && e.getMessage().contains("No space left on device")) { log.error("Shutdown due to lack of disk space."); System.exit(0); } continue; } - long t2 = System.currentTimeMillis(); - log.info( - "", - bw.getDisplayId(), - state.getMode(), - b.getShortHash(), - b.getNumber(), - b.getTransactionsList().size(), - importResult, - t2 - t1); switch (importResult) { case IMPORTED_BEST: @@ -232,28 +187,7 @@ public void run() { state.setBase(lastBlock); state.resetRepeated(); } else if (mode == PeerState.Mode.FORWARD) { - // continue - state.setBase(lastBlock); - // if the imported best block, switch back to normal mode - if (importResult == ImportResult.IMPORTED_BEST) { - state.setMode(PeerState.Mode.NORMAL); - // switch peers to NORMAL otherwise they may never switch back - for (PeerState peerState : peerStates.values()) { - if (peerState.getMode() != PeerState.Mode.NORMAL) { - peerState.setMode(PeerState.Mode.NORMAL); - peerState.setBase(b.getNumber()); - peerState.resetLastHeaderRequest(); - } - } - } - // if the maximum number of repeats is passed - // then the peer is stuck endlessly importing old blocks - // otherwise it would have found an IMPORTED block already - if (state.getRepeated() >= state.getMaxRepeats()) { - state.setMode(PeerState.Mode.NORMAL); - state.setBase(chain.getBestBlock().getNumber()); - state.resetLastHeaderRequest(); - } + forwardModeUpdate(state, lastBlock, importResult, b.getNumber()); } break; case NO_PARENT: @@ -291,4 +225,46 @@ public void run() { this.statis.update(this.chain.getBestBlock().getNumber()); } } + + private ImportResult importBlock(AionBlock b, String displayId, PeerState state) { + ImportResult importResult; + long t1 = System.currentTimeMillis(); + importResult = this.chain.tryToConnect(b); + long t2 = System.currentTimeMillis(); + log.info( + "", + displayId, + (state != null ? state.getMode() : PeerState.Mode.NORMAL), + b.getShortHash(), + b.getNumber(), + b.getTransactionsList().size(), + importResult, + t2 - t1); + return importResult; + } + + private void forwardModeUpdate(PeerState state, long lastBlock, ImportResult importResult, long blockNumber) { + // continue + state.setBase(lastBlock); + // if the imported best block, switch back to normal mode + if (importResult == ImportResult.IMPORTED_BEST) { + state.setMode(PeerState.Mode.NORMAL); + // switch peers to NORMAL otherwise they may never switch back + for (PeerState peerState : peerStates.values()) { + if (peerState.getMode() != PeerState.Mode.NORMAL) { + peerState.setMode(PeerState.Mode.NORMAL); + peerState.setBase(blockNumber); + peerState.resetLastHeaderRequest(); + } + } + } + // if the maximum number of repeats is passed + // then the peer is stuck endlessly importing old blocks + // otherwise it would have found an IMPORTED block already + if (state.getRepeated() >= state.getMaxRepeats()) { + state.setMode(PeerState.Mode.NORMAL); + state.setBase(chain.getBestBlock().getNumber()); + state.resetLastHeaderRequest(); + } + } } From bff2e1c3a95d27d3d8b8c3befe26b7aeb3cb3a25 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Tue, 29 May 2018 15:37:25 -0400 Subject: [PATCH 34/35] prune parameters as constants --- modMcf/src/org/aion/mcf/config/CfgDb.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/modMcf/src/org/aion/mcf/config/CfgDb.java b/modMcf/src/org/aion/mcf/config/CfgDb.java index 5a29267585..ae79157455 100644 --- a/modMcf/src/org/aion/mcf/config/CfgDb.java +++ b/modMcf/src/org/aion/mcf/config/CfgDb.java @@ -284,6 +284,19 @@ public CfgPrune getPrune() { return this.prune; } + /** + * Number of topmost blocks present in the database in TOP pruning mode. Information about these + * blocks is also kept in memory for later pruning. + */ + public static final int TOP_PRUNE_BLOCK_COUNT = 256; + /** + * Number of topmost blocks present in the database in SPREAD pruning mode. Information about + * these blocks is also kept in memory for later pruning. + */ + public static final int SPREAD_PRUNE_BLOCK_COUNT = 128; + /** At what frequency block states are being archived. */ + public static final int SPREAD_PRUNE_ARCHIVE_RATE = 10000; + public enum PruneOption { FULL, TOP, @@ -320,11 +333,11 @@ public void setPrune(String _prune_option) { switch (prune_option) { case TOP: // journal prune only - this.prune = new CfgPrune(128); + this.prune = new CfgPrune(TOP_PRUNE_BLOCK_COUNT); break; case SPREAD: // journal prune with archived states - this.prune = new CfgPrune(128, 10000); + this.prune = new CfgPrune(SPREAD_PRUNE_BLOCK_COUNT, SPREAD_PRUNE_ARCHIVE_RATE); break; case FULL: default: From e0ca459455c7307046b42491a613d57b53260bbe Mon Sep 17 00:00:00 2001 From: Alexandra Date: Wed, 30 May 2018 17:05:03 -0400 Subject: [PATCH 35/35] bugfix: reducing the heap space requirements for the state recovery --- .../src/org/aion/zero/impl/db/RecoveryUtils.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java b/modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java index fba8844118..1f0a06fb4b 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java +++ b/modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java @@ -368,6 +368,19 @@ public static void pruneOrRecoverState(String pruning_type) { "Rebuilding the main chain " + block.getNumber() + " blocks (may take a while) ..."); + + long topBlockNumber = block.getNumber(); + long blockNumber = 1000; + + // recover in increments of 1k blocks + while (blockNumber < topBlockNumber) { + block = store.getChainBlockByNumber(blockNumber); + chain.recoverWorldState(repo, block); + System.out.println("Finished with blocks up to " + blockNumber + "."); + blockNumber += 1000; + } + + block = store.getBestBlock(); chain.recoverWorldState(repo, block); repo.close();