diff --git a/modAionBase/src/org/aion/base/db/IKeyValueStore.java b/modAionBase/src/org/aion/base/db/IKeyValueStore.java index 5f2f8c00ba..d480cd619d 100644 --- a/modAionBase/src/org/aion/base/db/IKeyValueStore.java +++ b/modAionBase/src/org/aion/base/db/IKeyValueStore.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -42,14 +42,11 @@ /** * Functionality for a key-value cache allowing itemized updates. * - * @param - * the data type of the keys - * @param - * the data type of the values + * @param the data type of the keys + * @param the data type of the values * @author Alexandra Roatis - * @implNote For the underlying DB connection, if [isClosed() == true], then all - * function calls which are documented to throw RuntimeException, will - * throw a RuntimeException. + * @implNote For the underlying DB connection, if [isClosed() == true], then all function calls + * which are documented to throw RuntimeException, will throw a RuntimeException. */ public interface IKeyValueStore extends AutoCloseable { @@ -60,8 +57,7 @@ public interface IKeyValueStore extends AutoCloseable { * Returns if the DB is empty or not. * * @return True if number of keys > 0, false otherwise - * @throws RuntimeException - * if the data store is closed + * @throws RuntimeException if the data store is closed */ boolean isEmpty(); @@ -69,23 +65,18 @@ public interface IKeyValueStore extends AutoCloseable { * Returns the set of keys for the database. * * @return Set of keys - * @throws RuntimeException - * if the data store is closed - * @apiNote Returns an empty set if the database keys could not be - * retrieved. + * @throws RuntimeException if the data store is closed + * @apiNote Returns an empty set if the database keys could not be retrieved. */ Set keys(); /** - * get retrieves a value from the database, returning an optional, it is - * fulfilled if a value was able to be retrieved from the DB, otherwise the - * optional is empty + * get retrieves a value from the database, returning an optional, it is fulfilled if a value + * was able to be retrieved from the DB, otherwise the optional is empty * * @param k - * @throws RuntimeException - * if the data store is closed - * @throws IllegalArgumentException - * if the key is null + * @throws RuntimeException if the data store is closed + * @throws IllegalArgumentException if the key is null */ Optional get(K k); @@ -93,44 +84,34 @@ public interface IKeyValueStore extends AutoCloseable { // ------------------------------------------------------------------------------------- /** - * Places or updates a value into the cache at the corresponding key. Makes - * no guarantees about when the value is actually inserted into the - * underlying data store. + * Places or updates a value into the cache at the corresponding key. Makes no guarantees about + * when the value is actually inserted into the underlying data store. * - * @param k - * the key for the new entry - * @param v - * the value for the new entry - * @throws RuntimeException - * if the underlying data store is closed - * @throws IllegalArgumentException - * if the key is null - * @implNote The choice of when to push the changes to the data store is - * left up to the implementation. + * @param k the key for the new entry + * @param v the value for the new entry + * @throws RuntimeException if the underlying data store is closed + * @throws IllegalArgumentException if the key is null + * @implNote The choice of when to push the changes to the data store is left up to the + * implementation. * @apiNote Put must have the following properties: - *
    - *
  1. Creates a new entry in the cache, if the key-value pair does - * not exist in the cache or underlying data store.
    2. - *
  2. Updates the entry in the cache when the key-value pair - * already exists. - *
  3. Deletes the entry when given a {@code null} value.
    4. - *
+ *
    + *
  1. Creates a new entry in the cache, if the key-value pair does not exist in the cache + * or underlying data store. + *
  2. Updates the entry in the cache when the key-value pair already exists. + *
  3. Deletes the entry when given a {@code null} value. + *
*/ void put(K k, V v); /** - * Delete an entry from the cache, marking it for deletion inside the data - * store. Makes no guarantees about when the value is actually deleted from - * the underlying data store. + * Delete an entry from the cache, marking it for deletion inside the data store. Makes no + * guarantees about when the value is actually deleted from the underlying data store. * - * @param k - * the key of the entry to be deleted - * @throws RuntimeException - * if the underlying data store is closed - * @throws IllegalArgumentException - * if the key is null - * @implNote The choice of when to push the changes to the data store is - * left up to the implementation. + * @param k the key of the entry to be deleted + * @throws RuntimeException if the underlying data store is closed + * @throws IllegalArgumentException if the key is null + * @implNote The choice of when to push the changes to the data store is left up to the + * implementation. */ void delete(K k); @@ -138,37 +119,41 @@ public interface IKeyValueStore extends AutoCloseable { // --------------------------------------------------------------------------------------------------- /** - * Puts or updates the data store with the given key-value pairs, as - * follows: + * Puts or updates the data store with the given key-value pairs, as follows: + * *
    - *
  • if the key is present in the data store, the stored - * value is overwritten
    • - *
  • if the key is not present in the data store, the new - * key-value pair is stored
    • - *
  • if the value is null, the matching stored key will be - * deleted from the data store, or
    • + *
  • if the key is present in the data store, the stored value is overwritten + *
  • if the key is not present in the data store, the new key-value pair is + * stored + *
  • if the value is null, the matching stored key will be deleted from the + * data store, or *
* - * @param inputMap - * a {@link Map} of key-value pairs to be updated in the database - * @throws RuntimeException - * if the data store is closed - * @throws IllegalArgumentException - * if the map contains a null key + * @param inputMap a {@link Map} of key-value pairs to be updated in the database + * @throws RuntimeException if the data store is closed + * @throws IllegalArgumentException if the map contains a null key */ void putBatch(Map inputMap); - void putToBatch(byte[] key, byte[] value); + void putToBatch(K key, V value); + void commitBatch(); /** * Similar to delete, except operates on a list of keys * * @param keys - * @throws RuntimeException - * if the data store is closed - * @throws IllegalArgumentException - * if the collection contains a null key + * @throws RuntimeException if the data store is closed + * @throws IllegalArgumentException if the collection contains a null key */ void deleteBatch(Collection keys); + + /** + * Checks that the data store connection is open. Throws a {@link RuntimeException} if the data + * store connection is closed. + * + * @implNote Always do this check after acquiring a lock on the class/data. Otherwise it might + * produce inconsistent results due to lack of synchronization. + */ + void check(); } diff --git a/modAionBase/src/org/aion/base/db/IPruneConfig.java b/modAionBase/src/org/aion/base/db/IPruneConfig.java new file mode 100644 index 0000000000..3b93b1fb50 --- /dev/null +++ b/modAionBase/src/org/aion/base/db/IPruneConfig.java @@ -0,0 +1,37 @@ +package org.aion.base.db; + +/** + * Interface for pruning configuration parameters. + * + * @author Alexandra Roatis + */ +public interface IPruneConfig { + + /** + * Indicates if pruning should be enabled or disabled. + * + * @return {@code true} when pruning enabled, {@code false} when pruning disabled. + */ + boolean isEnabled(); + + /** + * Indicates if archiving should be enabled or disabled. + * + * @return {@code true} when archiving enabled, {@code false} when archiving disabled. + */ + boolean isArchived(); + + /** + * @return the number of topmost blocks for which the full data should be maintained on disk. + */ + int getCurrentCount(); + + /** + * Gets the rate at which blocks should be archived (for which the full data should be + * maintained on disk). Blocks that are exact multiples of the returned value should be + * persisted on disk, regardless of other pruning. + * + * @return integer value representing the archive rate + */ + int getArchiveRate(); +} diff --git a/modAionBase/src/org/aion/base/db/IRepositoryConfig.java b/modAionBase/src/org/aion/base/db/IRepositoryConfig.java index 02671bd267..7a315bbfd9 100644 --- a/modAionBase/src/org/aion/base/db/IRepositoryConfig.java +++ b/modAionBase/src/org/aion/base/db/IRepositoryConfig.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -37,19 +37,17 @@ import java.util.Properties; /** - * Represents a configuration interface accepted that should be accepted by the - * repository to implement necessary configs + * Represents a configuration interface accepted that should be accepted by the repository to + * implement necessary configs * * @author yao */ public interface IRepositoryConfig { - /** - * @return absolute path to the DB folder containing files - */ + /** @return absolute path to the DB folder containing files */ String getDbPath(); - int getPrune(); + IPruneConfig getPruneConfig(); IContractDetails contractDetailsImpl(); diff --git a/modAionImpl/src/org/aion/zero/impl/AionBlockchainImpl.java b/modAionImpl/src/org/aion/zero/impl/AionBlockchainImpl.java index 3078df2ff1..bdbd8bed66 100644 --- a/modAionImpl/src/org/aion/zero/impl/AionBlockchainImpl.java +++ b/modAionImpl/src/org/aion/zero/impl/AionBlockchainImpl.java @@ -121,7 +121,7 @@ public class AionBlockchainImpl implements IAionBlockchain { private A0BCConfig config; private long exitOn = Long.MAX_VALUE; - private IRepository repository; + private AionRepositoryImpl repository; private IRepositoryCache track; private TransactionStore transactionStore; private AionBlock bestBlock; @@ -218,7 +218,7 @@ private AionBlockchainImpl() { } protected AionBlockchainImpl(final A0BCConfig config, - final IRepository repository, + final AionRepositoryImpl repository, final ChainConfiguration chainConfig) { this.config = config; this.repository = repository; @@ -234,7 +234,7 @@ protected AionBlockchainImpl(final A0BCConfig config, this.parentHeaderValidator = this.chainConfiguration.createParentHeaderValidator(); this.blockHeaderValidator = this.chainConfiguration.createBlockHeaderValidator(); - this.transactionStore = ((AionRepositoryImpl) this.repository).getTransactionStore(); + this.transactionStore = this.repository.getTransactionStore(); this.minerCoinbase = this.config.getMinerCoinbase(); @@ -280,7 +280,7 @@ void setEventManager(IEventMgr eventManager) { } public AionBlockStore getBlockStore() { - return (AionBlockStore) repository.getBlockStore(); + return repository.getBlockStore(); } /** @@ -419,7 +419,7 @@ private static byte[] calcTxTrie(List transactions) { return txsState.getRootHash(); } - public IRepository getRepository() { + public AionRepositoryImpl getRepository() { return repository; } @@ -427,7 +427,7 @@ private State pushState(byte[] bestBlockHash) { State push = stateStack.push(new State()); this.bestBlock = getBlockStore().getBlockByHash(bestBlockHash); this.totalDifficulty = getBlockStore().getTotalDifficultyForHash(bestBlockHash); - this.repository = this.repository.getSnapshotTo(this.bestBlock.getStateRoot()); + this.repository = (AionRepositoryImpl) this.repository.getSnapshotTo(this.bestBlock.getStateRoot()); return push; } @@ -508,6 +508,17 @@ public boolean skipTryToConnect(long blockNumber) { return blockNumber > current + 32 || blockNumber < current - 32; } + /** + * Heuristic for skipping the call to tryToConnect with block number that was already pruned. + */ + public boolean isPruneRestricted(long blockNumber) { + // no restriction when not in TOP pruning mode + if (!repository.usesTopPruning()) { + return false; + } + return blockNumber < bestBlockNumber.get() - repository.getPruneBlockCount() + 1; + } + public synchronized ImportResult tryToConnect(final AionBlock block) { return tryToConnectInternal(block, System.currentTimeMillis() / THOUSAND_MS); } @@ -787,7 +798,7 @@ public synchronized AionBlockSummary add(AionBlock block, boolean rebuild) { track.rollback(); // block is bad so 'rollback' the state root to the original // state - ((AionRepositoryImpl) repository).setRoot(origRoot); + repository.setRoot(origRoot); } } @@ -800,7 +811,7 @@ public synchronized AionBlockSummary add(AionBlock block, boolean rebuild) { } transactionStore.flushBatch(); - ((AionRepositoryImpl) repository).commitBlock(block.getHeader()); + repository.commitBlock(block.getHeader()); if (LOG.isDebugEnabled()) LOG.debug("Block rebuilt: number: {}, hash: {}, TD: {}", block.getNumber(), block.getShortHash(), @@ -1095,7 +1106,7 @@ public synchronized void storeBlock(AionBlock block, List receipt } transactionStore.flushBatch(); - ((AionRepositoryImpl) repository).commitBlock(block.getHeader()); + repository.commitBlock(block.getHeader()); if (LOG.isDebugEnabled()) LOG.debug("Block saved: number: {}, hash: {}, TD: {}", block.getNumber(), block.getShortHash(), @@ -1160,7 +1171,7 @@ public void setTotalDifficulty(BigInteger totalDifficulty) { this.totalDifficulty = totalDifficulty; } - public void setRepository(IRepository repository) { + public void setRepository(AionRepositoryImpl repository) { this.repository = repository; } @@ -1349,7 +1360,7 @@ public List getListOfBodiesByHashes(List hashes) { private class State { - IRepository savedRepo = repository; + AionRepositoryImpl savedRepo = repository; AionBlock savedBest = bestBlock; BigInteger savedTD = totalDifficulty; } @@ -1380,11 +1391,14 @@ public synchronized boolean recoverWorldState(IRepository repository, AionBlock } long blockNumber = block.getNumber(); - LOG.info("Corrupt world state at block hash: {}, number: {}." + LOG.info("Pruned or corrupt world state at block hash: {}, number: {}." + " Looking for ancestor block with valid world state ...", block.getShortHash(), blockNumber); AionRepositoryImpl repo = (AionRepositoryImpl) repository; + // keeping track of the original root + byte[] originalRoot = repo.getRoot(); + Deque dirtyBlocks = new ArrayDeque<>(); // already known to be missing the state dirtyBlocks.push(block); @@ -1426,6 +1440,9 @@ public synchronized boolean recoverWorldState(IRepository repository, AionBlock // update the repository repo.flush(); + // setting the root back to its correct value + repo.syncToRoot(originalRoot); + // return a flag indicating if the recovery worked return repo.isValidRoot(block.getStateRoot()); } diff --git a/modAionImpl/src/org/aion/zero/impl/AionHub.java b/modAionImpl/src/org/aion/zero/impl/AionHub.java index 0328d86f9f..2695bd2fd8 100644 --- a/modAionImpl/src/org/aion/zero/impl/AionHub.java +++ b/modAionImpl/src/org/aion/zero/impl/AionHub.java @@ -283,9 +283,13 @@ private void loadBlockchain() { "Corrupt world state for genesis block hash: " + genesis.getShortHash() + ", number: " + genesis .getNumber() + "."); - buildGenesis(genesis); + AionHubUtils.buildGenesis(genesis, repository); - LOG.info("Rebuilding genesis block SUCCEEDED."); + if (repository.isValidRoot(genesis.getStateRoot())) { + LOG.info("Rebuilding genesis block SUCCEEDED."); + } else { + LOG.info("Rebuilding genesis block FAILED."); + } } recovered = this.blockchain.recoverWorldState(this.repository, bestBlock); @@ -338,7 +342,7 @@ private void loadBlockchain() { AionGenesis genesis = cfg.getGenesis(); - buildGenesis(genesis); + AionHubUtils.buildGenesis(genesis, repository); blockchain.setBestBlock(genesis); blockchain.setTotalDifficulty(genesis.getDifficultyBI()); @@ -394,30 +398,6 @@ private void loadBlockchain() { // this.repository.getBlockStore().load(); } - private void buildGenesis(AionGenesis genesis) { - // initialization section for network balance contract - IRepositoryCache track = repository.startTracking(); - - Address networkBalanceAddress = PrecompiledContracts.totalCurrencyAddress; - track.createAccount(networkBalanceAddress); - - for (Map.Entry addr : genesis.getNetworkBalances().entrySet()) { - track.addStorageRow( - networkBalanceAddress, - new DataWord(addr.getKey()), - new DataWord(addr.getValue())); - } - - for (Address addr : genesis.getPremine().keySet()) { - track.createAccount(addr); - track.addBalance(addr, genesis.getPremine().get(addr).getBalance()); - } - track.flush(); - - this.repository.commitBlock(genesis.getHeader()); - this.repository.getBlockStore().saveBlock(genesis, genesis.getCumulativeDifficulty(), true); - } - public void close() { LOG.info(""); diff --git a/modAionImpl/src/org/aion/zero/impl/AionHubUtils.java b/modAionImpl/src/org/aion/zero/impl/AionHubUtils.java new file mode 100644 index 0000000000..1beab75533 --- /dev/null +++ b/modAionImpl/src/org/aion/zero/impl/AionHubUtils.java @@ -0,0 +1,65 @@ +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: + * Aion foundation. + ******************************************************************************/ +package org.aion.zero.impl; + +import java.math.BigInteger; +import java.util.Map; +import org.aion.base.db.IRepositoryCache; +import org.aion.base.type.Address; +import org.aion.mcf.vm.types.DataWord; +import org.aion.vm.PrecompiledContracts; +import org.aion.zero.impl.db.AionRepositoryImpl; + +/** {@link AionHub} functionality where a full instantiation of the class is not desirable. */ +public class AionHubUtils { + + public static void buildGenesis(AionGenesis genesis, AionRepositoryImpl repository) { + // initialization section for network balance contract + IRepositoryCache track = repository.startTracking(); + + Address networkBalanceAddress = PrecompiledContracts.totalCurrencyAddress; + track.createAccount(networkBalanceAddress); + + for (Map.Entry addr : genesis.getNetworkBalances().entrySet()) { + track.addStorageRow( + networkBalanceAddress, + new DataWord(addr.getKey()), + new DataWord(addr.getValue())); + } + + for (Address addr : genesis.getPremine().keySet()) { + track.createAccount(addr); + track.addBalance(addr, genesis.getPremine().get(addr).getBalance()); + } + track.flush(); + + repository.commitBlock(genesis.getHeader()); + repository.getBlockStore().saveBlock(genesis, genesis.getCumulativeDifficulty(), true); + } +} diff --git a/modAionImpl/src/org/aion/zero/impl/StandaloneBlockchain.java b/modAionImpl/src/org/aion/zero/impl/StandaloneBlockchain.java index 5935942ced..dc4374cb89 100644 --- a/modAionImpl/src/org/aion/zero/impl/StandaloneBlockchain.java +++ b/modAionImpl/src/org/aion/zero/impl/StandaloneBlockchain.java @@ -25,6 +25,7 @@ import java.math.BigInteger; import java.util.*; import org.aion.base.db.IContractDetails; +import org.aion.base.db.IPruneConfig; import org.aion.base.db.IRepositoryCache; import org.aion.base.db.IRepositoryConfig; import org.aion.base.type.Address; @@ -33,6 +34,7 @@ import org.aion.crypto.ECKeyFac; import org.aion.db.impl.DBVendor; import org.aion.db.impl.DatabaseFactory; +import org.aion.mcf.config.CfgPrune; import org.aion.mcf.core.AccountState; import org.aion.mcf.core.ImportResult; import org.aion.mcf.valid.BlockHeaderValidator; @@ -68,8 +70,8 @@ public String getDbPath() { } @Override - public int getPrune() { - return -1; + public IPruneConfig getPruneConfig() { + return new CfgPrune(false); } @Override @@ -203,8 +205,8 @@ public String getDbPath() { } @Override - public int getPrune() { - return -1; + public IPruneConfig getPruneConfig() { + return new CfgPrune(false); } @Override @@ -336,8 +338,8 @@ public AbstractEnergyStrategyLimit getEnergyLimitStrategy() { // TODO: violates abstraction, consider adding to interface after // stable ((AionRepositoryImpl) bc.getRepository()).commitBlock(genesis.getHeader()); - ((AionBlockStore) bc.getRepository().getBlockStore()).saveBlock(genesis, genesis.getDifficultyBI(), - true); + ((AionBlockStore) bc.getRepository().getBlockStore()) + .saveBlock(genesis, genesis.getCumulativeDifficulty(), true); bc.setBestBlock(genesis); bc.setTotalDifficulty(genesis.getDifficultyBI()); diff --git a/modAionImpl/src/org/aion/zero/impl/cli/Cli.java b/modAionImpl/src/org/aion/zero/impl/cli/Cli.java index c7689ecad4..4acfa1d171 100644 --- a/modAionImpl/src/org/aion/zero/impl/cli/Cli.java +++ b/modAionImpl/src/org/aion/zero/impl/cli/Cli.java @@ -132,6 +132,20 @@ public int call(final String[] args, final Cfg cfg) { } } break; + case "--state": { + String pruning_type = "full"; + if (args.length >= 2) { + pruning_type = args[1]; + } + try { + RecoveryUtils.pruneOrRecoverState(pruning_type); + } catch (Throwable t) { + System.out.println("Reorganizing the state storage FAILED due to:"); + t.printStackTrace(); + return 1; + } + break; + } case "--dump-state-size": long block_count = 2L; diff --git a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java index 057312da49..61ad71f1d8 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java +++ b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -19,11 +19,15 @@ * * Contributors: * Aion foundation. - * ******************************************************************************/ - package org.aion.zero.impl.db; +import static org.aion.base.util.ByteUtil.EMPTY_BYTE_ARRAY; +import static org.aion.crypto.HashUtil.EMPTY_TRIE_HASH; + +import java.io.File; +import java.math.BigInteger; +import java.util.*; import org.aion.base.db.*; import org.aion.base.type.Address; import org.aion.base.util.Hex; @@ -42,28 +46,19 @@ import org.aion.zero.types.AionTransaction; import org.aion.zero.types.AionTxReceipt; -import java.io.File; -import java.math.BigInteger; -import java.util.*; - -import static org.aion.base.util.ByteUtil.EMPTY_BYTE_ARRAY; -import static org.aion.crypto.HashUtil.EMPTY_TRIE_HASH; - -/** - * Has direct database connection. - */ -public class AionRepositoryImpl extends AbstractRepository { +/** Has direct database connection. */ +public class AionRepositoryImpl + extends AbstractRepository { private TransactionStore transactionStore; /** * used by getSnapShotTo * - * @ATTENTION: when do snap shot, another instance will be created. Make - * sure it is used only by getSnapShotTo + *

@ATTENTION: when do snap shot, another instance will be created. Make sure it is used only + * by getSnapShotTo */ - protected AionRepositoryImpl() { - } + protected AionRepositoryImpl() {} protected AionRepositoryImpl(IRepositoryConfig repoConfig) { this.cfg = repoConfig; @@ -74,16 +69,13 @@ private static class AionRepositoryImplHolder { // configuration private static CfgAion config = CfgAion.inst(); // repository singleton instance - private final static AionRepositoryImpl inst = new AionRepositoryImpl( - new RepositoryConfig(new File(config.getBasePath(), config.getDb().getPath()).getAbsolutePath(), - config.getDb().getPrune() > 0 ? - // if the value is smaller than backward step - // there is the risk of importing state-less blocks after reboot - (128 > config.getDb().getPrune() ? 128 : config.getDb().getPrune()) : - // negative value => pruning disabled - config.getDb().getPrune(), - ContractDetailsAion.getInstance(), - config.getDb())); + private static final AionRepositoryImpl inst = + new AionRepositoryImpl( + new RepositoryConfig( + new File(config.getBasePath(), config.getDb().getPath()) + .getAbsolutePath(), + ContractDetailsAion.getInstance(), + config.getDb())); } public static AionRepositoryImpl inst() { @@ -99,8 +91,9 @@ private void init() { initializeDatabasesAndCaches(); // Setup the cache for transaction data source. - this.transactionStore = new TransactionStore<>(transactionDatabase, - AionTransactionStoreSerializer.serializer); + this.transactionStore = + new TransactionStore<>( + transactionDatabase, AionTransactionStoreSerializer.serializer); // Setup block store. this.blockStore = new AionBlockStore(indexDatabase, blockDatabase, checkIntegrity); @@ -112,19 +105,18 @@ private void init() { } } - /** - * @implNote The transaction store is not locked within the repository implementation. - */ + /** @implNote The transaction store is not locked within the repository implementation. */ public TransactionStore getTransactionStore() { return this.transactionStore; } private Trie createStateTrie() { - return new SecureTrie(stateDSPrune).withPruningEnabled(pruneBlockCount > 0); + return new SecureTrie(stateDSPrune).withPruningEnabled(pruneEnabled); } @Override - public void updateBatch(Map stateCache, + public void updateBatch( + Map stateCache, Map> detailsCache) { rwLock.writeLock().lock(); @@ -153,17 +145,19 @@ public void updateBatch(Map stateCache, updateAccountState(address, accountState); if (LOG.isTraceEnabled()) { - LOG.trace("update: [{}],nonce: [{}] balance: [{}] [{}]", - Hex.toHexString(address.toBytes()), - accountState.getNonce(), - accountState.getBalance(), - Hex.toHexString(contractDetails.getStorageHash())); + LOG.trace( + "update: [{}],nonce: [{}] balance: [{}] [{}]", + Hex.toHexString(address.toBytes()), + accountState.getNonce(), + accountState.getBalance(), + Hex.toHexString(contractDetails.getStorageHash())); } } continue; } - ContractDetailsCacheImpl contractDetailsCache = (ContractDetailsCacheImpl) contractDetails; + ContractDetailsCacheImpl contractDetailsCache = + (ContractDetailsCacheImpl) contractDetails; if (contractDetailsCache.origContract == null) { contractDetailsCache.origContract = this.cfg.contractDetailsImpl(); @@ -171,7 +165,8 @@ public void updateBatch(Map stateCache, contractDetailsCache.origContract.setAddress(address); } catch (Exception e) { e.printStackTrace(); - LOG.error("contractDetailsCache setAddress exception [{}]", e.toString()); + LOG.error( + "contractDetailsCache setAddress exception [{}]", e.toString()); } contractDetailsCache.commit(); @@ -188,11 +183,12 @@ public void updateBatch(Map stateCache, updateAccountState(address, accountState); if (LOG.isTraceEnabled()) { - LOG.trace("update: [{}],nonce: [{}] balance: [{}] [{}]", - Hex.toHexString(address.toBytes()), - accountState.getNonce(), - accountState.getBalance(), - Hex.toHexString(contractDetails.getStorageHash())); + LOG.trace( + "update: [{}],nonce: [{}] balance: [{}] [{}]", + Hex.toHexString(address.toBytes()), + accountState.getNonce(), + accountState.getBalance(), + Hex.toHexString(contractDetails.getStorageHash())); } } } @@ -205,10 +201,9 @@ public void updateBatch(Map stateCache, } } - /** - * @implNote The method calling this method must handle the locking. - */ - private void updateContractDetails(final Address address, final IContractDetails contractDetails) { + /** @implNote The method calling this method must handle the locking. */ + private void updateContractDetails( + final Address address, final IContractDetails contractDetails) { // locked by calling method detailsDS.update(address, contractDetails); } @@ -369,9 +364,7 @@ public BigInteger getNonce(Address address) { return (account == null) ? BigInteger.ZERO : account.getNonce(); } - /** - * @implNote The method calling this method must handle the locking. - */ + /** @implNote The method calling this method must handle the locking. */ private void updateAccountState(Address address, AccountState accountState) { // locked by calling method worldState.update(address.toBytes(), accountState.getEncoded()); @@ -379,10 +372,10 @@ private void updateAccountState(Address address, AccountState accountState) { /** * @inheritDoc - * @implNote Any other method calling this can rely on the fact that - * the contract details returned is a newly created object by {@link IContractDetails#getSnapshotTo(byte[])}. - * Since this querying method it locked, the methods calling it - * may not need to be locked or synchronized, depending on the specific use case. + * @implNote Any other method calling this can rely on the fact that the contract details + * returned is a newly created object by {@link IContractDetails#getSnapshotTo(byte[])}. + * Since this querying method it locked, the methods calling it may not need to be locked + * or synchronized, depending on the specific use case. */ @Override public IContractDetails getContractDetails(Address address) { @@ -423,10 +416,9 @@ public boolean hasContractDetails(Address address) { /** * @inheritDoc - * @implNote Any other method calling this can rely on the fact that - * the account state returned is a newly created object. - * Since this querying method it locked, the methods calling it - * may not need to be locked or synchronized, depending on the specific use case. + * @implNote Any other method calling this can rely on the fact that the account state returned + * is a newly created object. Since this querying method it locked, the methods calling it + * may not need to be locked or synchronized, depending on the specific use case. */ @Override public AccountState getAccountState(Address address) { @@ -439,7 +431,8 @@ public AccountState getAccountState(Address address) { if (accountData.length != 0) { result = new AccountState(accountData); - LOG.debug("New AccountSate [{}], State [{}]", address.toString(), result.toString()); + LOG.debug( + "New AccountSate [{}], State [{}]", address.toString(), result.toString()); } return result; } finally { @@ -453,11 +446,13 @@ public boolean hasAccountState(Address address) { } /** - * @implNote The loaded objects are fresh copies of the original account - * state and contract details. + * @implNote The loaded objects are fresh copies of the original account state and contract + * details. */ @Override - public void loadAccountState(Address address, Map cacheAccounts, + public void loadAccountState( + Address address, + Map cacheAccounts, Map> cacheDetails) { AccountState account = getAccountState(address); @@ -490,13 +485,8 @@ public void setRoot(byte[] root) { } } - public void setPruneBlockCount(long pruneBlockCount) { - rwLock.writeLock().lock(); - try { - this.pruneBlockCount = pruneBlockCount; - } finally { - rwLock.writeLock().unlock(); - } + public long getPruneBlockCount() { + return this.pruneBlockCount; } public void commitBlock(A0BlockHeader blockHeader) { @@ -506,9 +496,16 @@ public void commitBlock(A0BlockHeader blockHeader) { worldState.sync(); detailsDS.syncLargeStorage(); - if (pruneBlockCount > 0) { - stateDSPrune.storeBlockChanges(blockHeader); - detailsDS.getStorageDSPrune().storeBlockChanges(blockHeader); + if (pruneEnabled) { + if (blockHeader.getNumber() % archiveRate == 0 && stateDSPrune.isArchiveEnabled()) { + // archive block + worldState.saveDiffStateToDatabase( + blockHeader.getStateRoot(), stateDSPrune.getArchiveSource()); + } + stateDSPrune.storeBlockChanges(blockHeader.getHash(), blockHeader.getNumber()); + detailsDS + .getStorageDSPrune() + .storeBlockChanges(blockHeader.getHash(), blockHeader.getNumber()); pruneBlocks(blockHeader); } } finally { @@ -524,14 +521,22 @@ private void pruneBlocks(A0BlockHeader curBlock) { byte[] pruneBlockHash = blockStore.getBlockHashByNumber(pruneBlockNumber); if (pruneBlockHash != null) { A0BlockHeader header = blockStore.getBlockByHash(pruneBlockHash).getHeader(); - stateDSPrune.prune(header); - detailsDS.getStorageDSPrune().prune(header); + stateDSPrune.prune(header.getHash(), header.getNumber()); + detailsDS.getStorageDSPrune().prune(header.getHash(), header.getNumber()); } } } bestBlockNumber = curBlock.getNumber(); } + /** + * @return {@code true} when pruning is enabled and archiving is disabled, {@code false} + * otherwise + */ + public boolean usesTopPruning() { + return pruneEnabled && !stateDSPrune.isArchiveEnabled(); + } + public Trie getWorldState() { return worldState; } @@ -545,8 +550,14 @@ public IRepository getSnapshotTo(byte[] root) { repo.blockStore = blockStore; repo.cfg = cfg; repo.stateDatabase = this.stateDatabase; + repo.stateWithArchive = this.stateWithArchive; repo.stateDSPrune = this.stateDSPrune; + + // pruning config + repo.pruneEnabled = this.pruneEnabled; repo.pruneBlockCount = this.pruneBlockCount; + repo.archiveRate = this.archiveRate; + repo.detailsDS = this.detailsDS; repo.isSnapshot = true; @@ -597,10 +608,7 @@ public void removeTxBatch(Set clearTxSet, boolean isPool) { } } - /** - * This function cannot for any reason fail, otherwise we may have dangling - * file IO locks - */ + /** This function cannot for any reason fail, otherwise we may have dangling file IO locks */ @Override public void close() { rwLock.writeLock().lock(); @@ -625,6 +633,16 @@ public void close() { LOGGEN.error("Exception occurred while closing the state database.", e); } + try { + if (stateArchiveDatabase != null) { + stateArchiveDatabase.close(); + LOGGEN.info("State archive database closed."); + stateArchiveDatabase = null; + } + } catch (Exception e) { + LOGGEN.error("Exception occurred while closing the state archive database.", e); + } + try { if (transactionDatabase != null) { transactionDatabase.close(); @@ -662,7 +680,8 @@ public void close() { pendingTxCacheDatabase = null; } } catch (Exception e) { - LOGGEN.error("Exception occurred while closing the pendingTxCacheDatabase store.", e); + LOGGEN.error( + "Exception occurred while closing the pendingTxCacheDatabase store.", e); } } finally { rwLock.writeLock().unlock(); @@ -670,12 +689,11 @@ public void close() { } /** - * Retrieves the underlying state database that sits below all caches. This - * is usually provided by {@link org.aion.db.impl.leveldb.LevelDB} or - * {@link org.aion.db.impl.leveldb.LevelDB}. - *

- * Note that referencing the state database directly is unsafe, and should - * only be used for debugging and testing purposes. + * Retrieves the underlying state database that sits below all caches. This is usually provided + * by {@link org.aion.db.impl.leveldb.LevelDB} or {@link org.aion.db.impl.leveldb.LevelDB}. + * + *

Note that referencing the state database directly is unsafe, and should only be used for + * debugging and testing purposes. * * @return */ @@ -683,13 +701,16 @@ public IByteArrayKeyValueDatabase getStateDatabase() { return this.stateDatabase; } + public IByteArrayKeyValueDatabase getStateArchiveDatabase() { + return this.stateArchiveDatabase; + } + /** - * Retrieves the underlying details database that sits below all caches. - * This is usually provided by {@link org.aion.db.impl.mockdb.MockDB} - * or {@link org.aion.db.impl.mockdb.MockDB}. - *

- * Note that referencing the state database directly is unsafe, and should - * only be used for debugging and testing purposes. + * Retrieves the underlying details database that sits below all caches. This is usually + * provided by {@link org.aion.db.impl.mockdb.MockDB} or {@link org.aion.db.impl.mockdb.MockDB}. + * + *

Note that referencing the state database directly is unsafe, and should only be used for + * debugging and testing purposes. * * @return */ @@ -697,17 +718,20 @@ public IByteArrayKeyValueDatabase getDetailsDatabase() { return this.detailsDatabase; } - /** - * For testing. - */ + /** For testing. */ public IByteArrayKeyValueDatabase getIndexDatabase() { return this.indexDatabase; } @Override public String toString() { - return "AionRepositoryImpl{ identityHashCode=" + System.identityHashCode(this) + ", " + // - "databaseGroupSize=" + (databaseGroup == null ? 0 : databaseGroup.size()) + '}'; + return "AionRepositoryImpl{ identityHashCode=" + + System.identityHashCode(this) + + ", " + + // + "databaseGroupSize=" + + (databaseGroup == null ? 0 : databaseGroup.size()) + + '}'; } @Override diff --git a/modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java b/modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java index fa5590f630..1f0a06fb4b 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java +++ b/modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -22,27 +22,29 @@ ******************************************************************************/ package org.aion.zero.impl.db; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import org.aion.base.type.IBlock; import org.aion.log.AionLoggerFactory; +import org.aion.mcf.config.CfgDb; import org.aion.mcf.db.IBlockStoreBase; import org.aion.zero.impl.AionBlockchainImpl; +import org.aion.zero.impl.AionGenesis; +import org.aion.zero.impl.AionHubUtils; import org.aion.zero.impl.config.CfgAion; import org.aion.zero.impl.core.IAionBlockchain; import org.aion.zero.impl.types.AionBlock; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - public class RecoveryUtils { public enum Status { - SUCCESS, FAILURE, ILLEGAL_ARGUMENT + SUCCESS, + FAILURE, + ILLEGAL_ARGUMENT } - /** - * Used by the CLI call. - */ + /** Used by the CLI call. */ public static Status revertTo(long nbBlock) { // ensure mining is disabled CfgAion cfg = CfgAion.inst(); @@ -66,9 +68,7 @@ public static Status revertTo(long nbBlock) { return status; } - /** - * Used by the CLI call. - */ + /** Used by the CLI call. */ public static void pruneAndCorrect() { // ensure mining is disabled CfgAion cfg = CfgAion.inst(); @@ -101,9 +101,7 @@ public static void pruneAndCorrect() { blockchain.getRepository().close(); } - /** - * Used by the CLI call. - */ + /** Used by the CLI call. */ public static void dbCompact() { // ensure mining is disabled CfgAion cfg = CfgAion.inst(); @@ -126,9 +124,7 @@ public static void dbCompact() { repository.close(); } - /** - * Used by the CLI call. - */ + /** Used by the CLI call. */ public static void dumpBlocks(long count) { // ensure mining is disabled CfgAion cfg = CfgAion.inst(); @@ -161,9 +157,7 @@ public static void dumpBlocks(long count) { repository.close(); } - /** - * Used by internal world state recovery method. - */ + /** Used by internal world state recovery method. */ public static Status revertTo(IAionBlockchain blockchain, long nbBlock) { IBlockStoreBase store = blockchain.getBlockStore(); @@ -175,12 +169,15 @@ public static Status revertTo(IAionBlockchain blockchain, long nbBlock) { long nbBestBlock = bestBlock.getNumber(); - System.out.println("Attempting to revert best block from " + nbBestBlock + " to " + nbBlock + " ..."); + System.out.println( + "Attempting to revert best block from " + nbBestBlock + " to " + nbBlock + " ..."); // exit with warning if the given block is larger or negative if (nbBlock < 0) { System.out.println( - "Negative values <" + nbBlock + "> cannot be interpreted as block numbers. Nothing to do."); + "Negative values <" + + nbBlock + + "> cannot be interpreted as block numbers. Nothing to do."); return Status.ILLEGAL_ARGUMENT; } if (nbBestBlock == 0) { @@ -189,13 +186,19 @@ public static Status revertTo(IAionBlockchain blockchain, long nbBlock) { } if (nbBlock == nbBestBlock) { System.out.println( - "The block " + nbBlock + " is the current best block stored in the database. Nothing to do."); + "The block " + + nbBlock + + " is the current best block stored in the database. Nothing to do."); return Status.ILLEGAL_ARGUMENT; } if (nbBlock > nbBestBlock) { - System.out.println("The block #" + nbBlock + " is greater than the current best block #" + nbBestBlock - + " stored in the database. " - + "Cannot move to that block without synchronizing with peers. Start Aion instance to sync."); + System.out.println( + "The block #" + + nbBlock + + " is greater than the current best block #" + + nbBestBlock + + " stored in the database. " + + "Cannot move to that block without synchronizing with peers. Start Aion instance to sync."); return Status.ILLEGAL_ARGUMENT; } @@ -224,7 +227,7 @@ public static void printStateTrieSize(long blockNumber) { AionRepositoryImpl repository = AionRepositoryImpl.inst(); AionBlockStore store = repository.getBlockStore(); - long topBlock = store.getMaxNumber(); + long topBlock = store.getBestBlock().getNumber(); if (topBlock < 0) { System.out.println("The database is empty. Cannot print block information."); return; @@ -244,17 +247,33 @@ public static void printStateTrieSize(long blockNumber) { stateRoot = block.getStateRoot(); try { System.out.println( - "Block hash: " + block.getShortHash() + ", number: " + block.getNumber() + ", tx count: " - + block.getTransactionsList().size() + ", state trie kv count = " + repository - .getWorldState().getTrieSize(stateRoot)); + "Block hash: " + + block.getShortHash() + + ", number: " + + block.getNumber() + + ", tx count: " + + block.getTransactionsList().size() + + ", state trie kv count = " + + repository.getWorldState().getTrieSize(stateRoot)); } catch (RuntimeException e) { System.out.println( - "Block hash: " + block.getShortHash() + ", number: " + block.getNumber() + ", tx count: " - + block.getTransactionsList().size() + ", state trie kv count threw exception: " + e - .getMessage()); + "Block hash: " + + block.getShortHash() + + ", number: " + + block.getNumber() + + ", tx count: " + + block.getTransactionsList().size() + + ", state trie kv count threw exception: " + + e.getMessage()); } } else { - System.out.println("Null block found at level " + targetBlock + "."); + long count = store.getBlocksByNumber(targetBlock).size(); + System.out.println( + "Null block found at level " + + targetBlock + + ". There " + + (count == 1 ? "is 1 block" : "are " + count + " blocks") + + " at this level. No main chain block found."); } targetBlock++; } @@ -296,9 +315,75 @@ public static void printStateTrieDump(long blockNumber) { } byte[] stateRoot = block.getStateRoot(); - System.out.println("\nBlock hash: " + block.getShortHash() + ", number: " + blockNumber + ", tx count: " + block - .getTransactionsList().size() + "\n\n" + repository.getWorldState().getTrieDump(stateRoot)); + System.out.println( + "\nBlock hash: " + + block.getShortHash() + + ", number: " + + blockNumber + + ", tx count: " + + block.getTransactionsList().size() + + "\n\n" + + repository.getWorldState().getTrieDump(stateRoot)); repository.close(); } + + public static void pruneOrRecoverState(String pruning_type) { + // ensure mining is disabled + CfgAion cfg = CfgAion.inst(); + cfg.fromXML(); + cfg.getConsensus().setMining(false); + + // setting pruning to the version requested + CfgDb.PruneOption option = CfgDb.PruneOption.fromValue(pruning_type); + cfg.getDb().setPrune(option.toString()); + + System.out.println("Reorganizing the state storage to " + option + " mode ..."); + + Map cfgLog = new HashMap<>(); + cfgLog.put("DB", "ERROR"); + cfgLog.put("CONS", "ERROR"); + + AionLoggerFactory.init(cfgLog); + + AionBlockchainImpl chain = AionBlockchainImpl.inst(); + AionRepositoryImpl repo = (AionRepositoryImpl) chain.getRepository(); + AionBlockStore store = repo.getBlockStore(); + + // dropping old state database + System.out.println("Deleting old data ..."); + repo.getStateDatabase().drop(); + if (pruning_type.equals("spread")) { + repo.getStateArchiveDatabase().drop(); + } + + // recover genesis + System.out.println("Rebuilding genesis block ..."); + AionGenesis genesis = cfg.getGenesis(); + AionHubUtils.buildGenesis(genesis, repo); + + // recover all blocks + AionBlock block = store.getBestBlock(); + System.out.println( + "Rebuilding the main chain " + + block.getNumber() + + " blocks (may take a while) ..."); + + long topBlockNumber = block.getNumber(); + long blockNumber = 1000; + + // recover in increments of 1k blocks + while (blockNumber < topBlockNumber) { + block = store.getChainBlockByNumber(blockNumber); + chain.recoverWorldState(repo, block); + System.out.println("Finished with blocks up to " + blockNumber + "."); + blockNumber += 1000; + } + + block = store.getBestBlock(); + chain.recoverWorldState(repo, block); + + repo.close(); + System.out.println("Reorganizing the state storage COMPLETE."); + } } diff --git a/modAionImpl/src/org/aion/zero/impl/db/RepositoryConfig.java b/modAionImpl/src/org/aion/zero/impl/db/RepositoryConfig.java index a2acc69722..c65456edcb 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/RepositoryConfig.java +++ b/modAionImpl/src/org/aion/zero/impl/db/RepositoryConfig.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -19,23 +19,21 @@ * * Contributors: * Aion foundation. - * ******************************************************************************/ - package org.aion.zero.impl.db; +import java.util.Map; +import java.util.Properties; import org.aion.base.db.DetailsProvider; import org.aion.base.db.IContractDetails; +import org.aion.base.db.IPruneConfig; import org.aion.base.db.IRepositoryConfig; import org.aion.mcf.config.CfgDb; -import java.util.Map; -import java.util.Properties; - public class RepositoryConfig implements IRepositoryConfig { private final String dbPath; - private final int prune; + private final IPruneConfig cfgPrune; private final DetailsProvider detailsProvider; private final Map cfg; @@ -45,8 +43,8 @@ public String getDbPath() { } @Override - public int getPrune() { - return prune; + public IPruneConfig getPruneConfig() { + return cfgPrune; } @Override @@ -63,13 +61,11 @@ public Properties getDatabaseConfig(String db_name) { return new Properties(prop); } - public RepositoryConfig(final String dbPath, - final int prune, - final DetailsProvider detailsProvider, - final CfgDb cfgDb) { + public RepositoryConfig( + final String dbPath, final DetailsProvider detailsProvider, final CfgDb cfgDb) { this.dbPath = dbPath; - this.prune = prune; this.detailsProvider = detailsProvider; this.cfg = cfgDb.asProperties(); + this.cfgPrune = cfgDb.getPrune(); } } diff --git a/modAionImpl/src/org/aion/zero/impl/sync/PeerState.java b/modAionImpl/src/org/aion/zero/impl/sync/PeerState.java index 36e8020e15..7d0d52fabc 100644 --- a/modAionImpl/src/org/aion/zero/impl/sync/PeerState.java +++ b/modAionImpl/src/org/aion/zero/impl/sync/PeerState.java @@ -3,42 +3,28 @@ public class PeerState { public enum Mode { - /** - * The peer is in main-chain; use normal syncing strategy. - */ + /** The peer is in main-chain; use normal syncing strategy. */ NORMAL, - /** - * The peer is in side-chain; sync backward to find the fork point. - */ + /** The peer is in side-chain; sync backward to find the fork point. */ BACKWARD, - /** - * The peer is in side-chain; sync forward to catch up. - */ + /** The peer is in side-chain; sync forward to catch up. */ FORWARD } // TODO: enforce rules on this public enum State { - /** - * The initial state. - */ + /** The initial state. */ INITIAL, - /** - * Status request, waiting for response. - */ + /** Status request, waiting for response. */ STATUS_REQUESTED, - /** - * Block headers request, waiting for response. - */ + /** Block headers request, waiting for response. */ HEADERS_REQUESTED, - /** - * Block bodies request, waiting for response. - */ + /** Block bodies request, waiting for response. */ BODIES_REQUESTED, } @@ -46,6 +32,11 @@ public enum State { private Mode mode; private long base; + // used in FORWARD mode to prevent endlessly importing EXISTing blocks + // compute how many times to go forward without importing a new block + private int repeated; + private int maxRepeats; + // The syncing status private State state; private long lastHeaderRequest; @@ -98,4 +89,33 @@ public void setLastHeaderRequest(long lastStatusRequest) { public void resetLastHeaderRequest() { this.lastHeaderRequest = 0; } + + public int getRepeated() { + return repeated; + } + + public void resetRepeated() { + this.repeated = 0; + } + + public void incRepeated() { + this.repeated++; + } + + /** + * This number is set based on the BACKWARD step size and the size of each requested batch in + * FORWARD mode. Passing the number of repeats allowed means that we have entered in the + * previous BACKWARD step. If that step would have been viable, we never would have made another + * step back, so it effectively ends the FORWARD pass. + * + * @return The number of times that a node in FORWARD mode can import only blocks that already + * EXIST. + */ + public int getMaxRepeats() { + return maxRepeats; + } + + public void setMaxRepeats(int maxRepeats) { + this.maxRepeats = maxRepeats; + } } diff --git a/modAionImpl/src/org/aion/zero/impl/sync/TaskGetHeaders.java b/modAionImpl/src/org/aion/zero/impl/sync/TaskGetHeaders.java index 7c8cf75f91..357bb20649 100644 --- a/modAionImpl/src/org/aion/zero/impl/sync/TaskGetHeaders.java +++ b/modAionImpl/src/org/aion/zero/impl/sync/TaskGetHeaders.java @@ -29,21 +29,20 @@ package org.aion.zero.impl.sync; -import org.aion.p2p.INode; -import org.aion.p2p.IP2pMgr; -import org.aion.zero.impl.sync.msg.ReqBlocksHeaders; -import org.slf4j.Logger; - import java.math.BigInteger; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Random; import java.util.stream.Collectors; +import org.aion.p2p.INode; +import org.aion.p2p.IP2pMgr; +import org.aion.zero.impl.sync.msg.ReqBlocksHeaders; +import org.slf4j.Logger; -/** - * @author chris - */ +import static org.aion.p2p.P2pConstant.BACKWARD_SYNC_STEP; + +/** @author chris */ final class TaskGetHeaders implements Runnable { private final IP2pMgr p2p; @@ -58,7 +57,12 @@ final class TaskGetHeaders implements Runnable { private final Random random = new Random(System.currentTimeMillis()); - TaskGetHeaders(IP2pMgr p2p, long selfNumber, BigInteger selfTd, Map peerStates, Logger log) { + TaskGetHeaders( + IP2pMgr p2p, + long selfNumber, + BigInteger selfTd, + Map peerStates, + Logger log) { this.p2p = p2p; this.selfNumber = selfNumber; this.selfTd = selfTd; @@ -78,8 +82,9 @@ public void run() { // higher td n.getTotalDifficulty() != null && n.getTotalDifficulty().compareTo(this.selfTd) >= 0 // not recently requested - && (now - 5000) > peerStates.computeIfAbsent(n.getIdHash(), k -> new PeerState(PeerState.Mode.NORMAL, selfNumber)).getLastHeaderRequest() - ) + && (now - 5000) > peerStates + .computeIfAbsent(n.getIdHash(), k -> new PeerState(PeerState.Mode.NORMAL, selfNumber)) + .getLastHeaderRequest()) .collect(Collectors.toList()); if (nodesFiltered.isEmpty()) { return; @@ -94,39 +99,51 @@ public void run() { // decide the start block number long from = 0; int size = 24; + + // depends on the number of blocks going BACKWARD + state.setMaxRepeats(BACKWARD_SYNC_STEP / size + 1); + switch (state.getMode()) { - case NORMAL: { - // update base block - state.setBase(selfNumber); - - // normal mode - long nodeNumber = node.getBestBlockNumber(); - if (nodeNumber >= selfNumber + 128) { - from = Math.max(1, selfNumber + 1 - 4); - } else if (nodeNumber >= selfNumber - 128) { - from = Math.max(1, selfNumber + 1 - 16); - } else { - // no need to request from this node. His TD is probably corrupted. - return; + case NORMAL: + { + // update base block + state.setBase(selfNumber); + + // normal mode + long nodeNumber = node.getBestBlockNumber(); + if (nodeNumber >= selfNumber + BACKWARD_SYNC_STEP) { + from = Math.max(1, selfNumber + 1 - 4); + } else if (nodeNumber >= selfNumber - BACKWARD_SYNC_STEP) { + from = Math.max(1, selfNumber + 1 - 16); + } else { + // no need to request from this node. His TD is probably corrupted. + return; + } + + break; + } + case BACKWARD: + { + // step back by 128 blocks + from = Math.max(1, state.getBase() - BACKWARD_SYNC_STEP); + break; + } + case FORWARD: + { + // start from base block + from = state.getBase() + 1; + break; } - - break; - } - case BACKWARD: { - // step back by 128 blocks - from = Math.max(1, state.getBase() - 128); - break; - } - case FORWARD: { - // start from base block - from = state.getBase() + 1; - break; - } } // send request if (log.isDebugEnabled()) { - log.debug("", state.getMode(), from, size, node.getIdShort()); + log.debug( + "", + state.getMode(), + from, + size, + node.getIdShort()); } ReqBlocksHeaders rbh = new ReqBlocksHeaders(from, size); this.p2p.send(node.getIdHash(), node.getIdShort(), rbh); diff --git a/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java b/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java index bb7577a2d2..3e579f5288 100644 --- a/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java +++ b/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java @@ -29,6 +29,11 @@ package org.aion.zero.impl.sync; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; import org.aion.base.util.ByteArrayWrapper; import org.aion.mcf.core.ImportResult; import org.aion.p2p.IP2pMgr; @@ -36,16 +41,12 @@ import org.aion.zero.impl.types.AionBlock; import org.slf4j.Logger; -import java.util.List; -import java.util.Map; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - /** - * @author chris * handle process of importing blocks to repo - * TODO: targeted send + * + *

TODO: targeted send + * + * @author chris */ final class TaskImportBlocks implements Runnable { @@ -73,8 +74,7 @@ final class TaskImportBlocks implements Runnable { final BlockingQueue downloadedBlocks, final Map importedBlockHashes, final Map peerStates, - final Logger log - ) { + final Logger log) { this.p2p = p2p; this.chain = _chain; this.start = _start; @@ -99,18 +99,56 @@ public void run() { List batch = bw.getBlocks().stream() .filter(b -> importedBlockHashes.get(ByteArrayWrapper.wrap(b.getHash())) == null) + .filter(b -> !chain.isPruneRestricted(b.getNumber())) .collect(Collectors.toList()); PeerState state = peerStates.get(bw.getNodeIdHash()); if (state == null) { - log.warn("This is not supposed to happen, but the peer is sending us blocks without ask"); + log.warn( + "This is not supposed to happen, but the peer is sending us blocks without ask"); + } + + ImportResult importResult = ImportResult.IMPORTED_NOT_BEST; + + // importing last block in batch to see if we can skip batch + if (state != null && state.getMode() == PeerState.Mode.FORWARD && !batch.isEmpty()) { + AionBlock b = batch.get(batch.size() - 1); + + try { + importResult = importBlock(b, bw.getDisplayId(), state); + } catch (Throwable e) { + log.error(" {}", e.toString()); + if (e.getMessage() != null && e.getMessage().contains("No space left on device")) { + log.error("Shutdown due to lack of disk space."); + System.exit(0); + } + continue; + } + + switch (importResult) { + case IMPORTED_BEST: + case IMPORTED_NOT_BEST: + case EXIST: + { + importedBlockHashes.put(ByteArrayWrapper.wrap(b.getHash()), true); + + long lastBlock = batch.get(batch.size() - 1).getNumber(); + + forwardModeUpdate(state, lastBlock, importResult, b.getNumber()); + + // since last import worked skipping the batch + batch.clear(); + log.info("Forward skip."); + break; + } + default: + break; + } } for (AionBlock b : batch) { - long t1 = System.currentTimeMillis(); - ImportResult importResult; try { - importResult = this.chain.tryToConnect(b); + importResult = importBlock(b, bw.getDisplayId(), state); } catch (Throwable e) { log.error(" {}", e.toString()); if (e.getMessage() != null && e.getMessage().contains("No space left on device")) { @@ -119,14 +157,7 @@ public void run() { } continue; } - long t2 = System.currentTimeMillis(); - log.info("", - bw.getDisplayId(), - b.getShortHash(), - b.getNumber(), - b.getTransactionsList().size(), - importResult, - t2 - t1); + switch (importResult) { case IMPORTED_BEST: case IMPORTED_NOT_BEST: @@ -154,14 +185,9 @@ public void run() { // we found the fork point state.setMode(PeerState.Mode.FORWARD); state.setBase(lastBlock); - + state.resetRepeated(); } else if (mode == PeerState.Mode.FORWARD) { - // continue - state.setBase(lastBlock); - // if the imported best block, switch back to normal mode - if (importResult == ImportResult.IMPORTED_BEST) { - state.setMode(PeerState.Mode.NORMAL); - } + forwardModeUpdate(state, lastBlock, importResult, b.getNumber()); } break; case NO_PARENT: @@ -176,6 +202,20 @@ public void run() { break; } } + + // if any block results in NO_PARENT, all subsequent blocks will too + if (importResult == ImportResult.NO_PARENT) { + log.debug("Stopped importing batch due to NO_PARENT result."); + break; + } + } + + if (state != null + && state.getMode() == PeerState.Mode.FORWARD + && importResult == ImportResult.EXIST) { + // increment the repeat count every time + // we finish a batch of imports with EXIST + state.incRepeated(); } if (state != null) { @@ -185,4 +225,46 @@ public void run() { this.statis.update(this.chain.getBestBlock().getNumber()); } } + + private ImportResult importBlock(AionBlock b, String displayId, PeerState state) { + ImportResult importResult; + long t1 = System.currentTimeMillis(); + importResult = this.chain.tryToConnect(b); + long t2 = System.currentTimeMillis(); + log.info( + "", + displayId, + (state != null ? state.getMode() : PeerState.Mode.NORMAL), + b.getShortHash(), + b.getNumber(), + b.getTransactionsList().size(), + importResult, + t2 - t1); + return importResult; + } + + private void forwardModeUpdate(PeerState state, long lastBlock, ImportResult importResult, long blockNumber) { + // continue + state.setBase(lastBlock); + // if the imported best block, switch back to normal mode + if (importResult == ImportResult.IMPORTED_BEST) { + state.setMode(PeerState.Mode.NORMAL); + // switch peers to NORMAL otherwise they may never switch back + for (PeerState peerState : peerStates.values()) { + if (peerState.getMode() != PeerState.Mode.NORMAL) { + peerState.setMode(PeerState.Mode.NORMAL); + peerState.setBase(blockNumber); + peerState.resetLastHeaderRequest(); + } + } + } + // if the maximum number of repeats is passed + // then the peer is stuck endlessly importing old blocks + // otherwise it would have found an IMPORTED block already + if (state.getRepeated() >= state.getMaxRepeats()) { + state.setMode(PeerState.Mode.NORMAL); + state.setBase(chain.getBestBlock().getNumber()); + state.resetLastHeaderRequest(); + } + } } diff --git a/modAionImpl/test/org/aion/db/AionContractDetailsTest.java b/modAionImpl/test/org/aion/db/AionContractDetailsTest.java index 4222d694fc..5c69f27978 100644 --- a/modAionImpl/test/org/aion/db/AionContractDetailsTest.java +++ b/modAionImpl/test/org/aion/db/AionContractDetailsTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -34,59 +34,61 @@ ******************************************************************************/ package org.aion.db; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.base.db.IContractDetails; +import org.aion.base.db.IPruneConfig; import org.aion.base.db.IRepositoryConfig; import org.aion.base.type.Address; import org.aion.base.util.ByteUtil; import org.aion.db.impl.DBVendor; import org.aion.db.impl.DatabaseFactory; -import org.aion.db.impl.leveldb.LevelDBConstants; +import org.aion.mcf.config.CfgPrune; import org.aion.mcf.vm.types.DataWord; import org.aion.zero.db.AionContractDetailsImpl; import org.aion.zero.impl.db.AionRepositoryImpl; import org.aion.zero.impl.db.ContractDetailsAion; import org.apache.commons.lang3.RandomUtils; -import org.junit.Ignore; import org.junit.Test; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - public class AionContractDetailsTest { - private static final int IN_MEMORY_STORAGE_LIMIT = 1000000; // CfgAion.inst().getDb().getDetailsInMemoryStorageLimit(); - - protected IRepositoryConfig repoConfig = new IRepositoryConfig() { - @Override - public String getDbPath() { - return ""; - } - - @Override - public int getPrune() { - return 0; - } - - @Override - public IContractDetails contractDetailsImpl() { - return ContractDetailsAion.createForTesting(0, 1000000).getDetails(); - } - - @Override - public Properties getDatabaseConfig(String db_name) { - Properties props = new Properties(); - props.setProperty(DatabaseFactory.Props.DB_TYPE, DBVendor.MOCKDB.toValue()); - props.setProperty(DatabaseFactory.Props.ENABLE_HEAP_CACHE, "false"); - return props; - } - }; - - private static IContractDetails deserialize(byte[] rlp, IByteArrayKeyValueDatabase externalStorage) { + private static final int IN_MEMORY_STORAGE_LIMIT = + 1000000; // CfgAion.inst().getDb().getDetailsInMemoryStorageLimit(); + + protected IRepositoryConfig repoConfig = + new IRepositoryConfig() { + @Override + public String getDbPath() { + return ""; + } + + @Override + public IPruneConfig getPruneConfig() { + return new CfgPrune(false); + } + + @Override + public IContractDetails contractDetailsImpl() { + return ContractDetailsAion.createForTesting(0, 1000000).getDetails(); + } + + @Override + public Properties getDatabaseConfig(String db_name) { + Properties props = new Properties(); + props.setProperty(DatabaseFactory.Props.DB_TYPE, DBVendor.MOCKDB.toValue()); + props.setProperty(DatabaseFactory.Props.ENABLE_HEAP_CACHE, "false"); + return props; + } + }; + + private static IContractDetails deserialize( + byte[] rlp, IByteArrayKeyValueDatabase externalStorage) { AionContractDetailsImpl result = new AionContractDetailsImpl(); result.setExternalStorageDataSource(externalStorage); result.decode(rlp); @@ -105,10 +107,11 @@ public void test_1() throws Exception { byte[] key_2 = ByteUtil.hexStringToBytes("222222"); byte[] val_2 = ByteUtil.hexStringToBytes("bbbbbb"); - AionContractDetailsImpl contractDetails = new AionContractDetailsImpl( - -1, //CfgAion.inst().getDb().getPrune(), - 1000000 //CfgAion.inst().getDb().getDetailsInMemoryStorageLimit() - ); + AionContractDetailsImpl contractDetails = + new AionContractDetailsImpl( + -1, // CfgAion.inst().getDb().getPrune(), + 1000000 // CfgAion.inst().getDb().getDetailsInMemoryStorageLimit() + ); contractDetails.setCode(code); contractDetails.put(new DataWord(key_1), new DataWord(val_1)); contractDetails.put(new DataWord(key_2), new DataWord(val_2)); @@ -117,20 +120,25 @@ public void test_1() throws Exception { AionContractDetailsImpl contractDetails_ = new AionContractDetailsImpl(data); - assertEquals(ByteUtil.toHexString(code), - ByteUtil.toHexString(contractDetails_.getCode())); + assertEquals(ByteUtil.toHexString(code), ByteUtil.toHexString(contractDetails_.getCode())); - assertEquals(ByteUtil.toHexString(val_1), - ByteUtil.toHexString(contractDetails_.get(new DataWord(key_1)).getNoLeadZeroesData())); + assertEquals( + ByteUtil.toHexString(val_1), + ByteUtil.toHexString( + contractDetails_.get(new DataWord(key_1)).getNoLeadZeroesData())); - assertEquals(ByteUtil.toHexString(val_2), - ByteUtil.toHexString(contractDetails_.get(new DataWord(key_2)).getNoLeadZeroesData())); + assertEquals( + ByteUtil.toHexString(val_2), + ByteUtil.toHexString( + contractDetails_.get(new DataWord(key_2)).getNoLeadZeroesData())); } @Test public void test_2() throws Exception { - byte[] code = ByteUtil.hexStringToBytes("7c0100000000000000000000000000000000000000000000000000000000600035046333d546748114610065578063430fe5f01461007c5780634d432c1d1461008d578063501385b2146100b857806357eb3b30146100e9578063dbc7df61146100fb57005b6100766004356024356044356102f0565b60006000f35b61008760043561039e565b60006000f35b610098600435610178565b8073ffffffffffffffffffffffffffffffffffffffff1660005260206000f35b6100c96004356024356044356101a0565b8073ffffffffffffffffffffffffffffffffffffffff1660005260206000f35b6100f1610171565b8060005260206000f35b610106600435610133565b8360005282602052816040528073ffffffffffffffffffffffffffffffffffffffff1660605260806000f35b5b60006020819052908152604090208054600182015460028301546003909301549192909173ffffffffffffffffffffffffffffffffffffffff1684565b5b60015481565b5b60026020526000908152604090205473ffffffffffffffffffffffffffffffffffffffff1681565b73ffffffffffffffffffffffffffffffffffffffff831660009081526020819052604081206002015481908302341080156101fe575073ffffffffffffffffffffffffffffffffffffffff8516600090815260208190526040812054145b8015610232575073ffffffffffffffffffffffffffffffffffffffff85166000908152602081905260409020600101548390105b61023b57610243565b3391506102e8565b6101966103ca60003973ffffffffffffffffffffffffffffffffffffffff3381166101965285166101b68190526000908152602081905260408120600201546101d6526101f68490526102169080f073ffffffffffffffffffffffffffffffffffffffff8616600090815260208190526040902060030180547fffffffffffffffffffffffff0000000000000000000000000000000000000000168217905591508190505b509392505050565b73ffffffffffffffffffffffffffffffffffffffff33166000908152602081905260408120548190821461032357610364565b60018054808201909155600090815260026020526040902080547fffffffffffffffffffffffff000000000000000000000000000000000000000016331790555b50503373ffffffffffffffffffffffffffffffffffffffff1660009081526020819052604090209081556001810192909255600290910155565b3373ffffffffffffffffffffffffffffffffffffffff166000908152602081905260409020600201555600608061019660043960048051602451604451606451600080547fffffffffffffffffffffffff0000000000000000000000000000000000000000908116909517815560018054909516909317909355600355915561013390819061006390396000f3007c0100000000000000000000000000000000000000000000000000000000600035046347810fe381146100445780637e4a1aa81461005557806383d2421b1461006957005b61004f6004356100ab565b60006000f35b6100636004356024356100fc565b60006000f35b61007460043561007a565b60006000f35b6001543373ffffffffffffffffffffffffffffffffffffffff9081169116146100a2576100a8565b60078190555b50565b73ffffffffffffffffffffffffffffffffffffffff8116600090815260026020526040902080547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0016600117905550565b6001543373ffffffffffffffffffffffffffffffffffffffff9081169116146101245761012f565b600582905560068190555b505056"); + byte[] code = + ByteUtil.hexStringToBytes( + "7c0100000000000000000000000000000000000000000000000000000000600035046333d546748114610065578063430fe5f01461007c5780634d432c1d1461008d578063501385b2146100b857806357eb3b30146100e9578063dbc7df61146100fb57005b6100766004356024356044356102f0565b60006000f35b61008760043561039e565b60006000f35b610098600435610178565b8073ffffffffffffffffffffffffffffffffffffffff1660005260206000f35b6100c96004356024356044356101a0565b8073ffffffffffffffffffffffffffffffffffffffff1660005260206000f35b6100f1610171565b8060005260206000f35b610106600435610133565b8360005282602052816040528073ffffffffffffffffffffffffffffffffffffffff1660605260806000f35b5b60006020819052908152604090208054600182015460028301546003909301549192909173ffffffffffffffffffffffffffffffffffffffff1684565b5b60015481565b5b60026020526000908152604090205473ffffffffffffffffffffffffffffffffffffffff1681565b73ffffffffffffffffffffffffffffffffffffffff831660009081526020819052604081206002015481908302341080156101fe575073ffffffffffffffffffffffffffffffffffffffff8516600090815260208190526040812054145b8015610232575073ffffffffffffffffffffffffffffffffffffffff85166000908152602081905260409020600101548390105b61023b57610243565b3391506102e8565b6101966103ca60003973ffffffffffffffffffffffffffffffffffffffff3381166101965285166101b68190526000908152602081905260408120600201546101d6526101f68490526102169080f073ffffffffffffffffffffffffffffffffffffffff8616600090815260208190526040902060030180547fffffffffffffffffffffffff0000000000000000000000000000000000000000168217905591508190505b509392505050565b73ffffffffffffffffffffffffffffffffffffffff33166000908152602081905260408120548190821461032357610364565b60018054808201909155600090815260026020526040902080547fffffffffffffffffffffffff000000000000000000000000000000000000000016331790555b50503373ffffffffffffffffffffffffffffffffffffffff1660009081526020819052604090209081556001810192909255600290910155565b3373ffffffffffffffffffffffffffffffffffffffff166000908152602081905260409020600201555600608061019660043960048051602451604451606451600080547fffffffffffffffffffffffff0000000000000000000000000000000000000000908116909517815560018054909516909317909355600355915561013390819061006390396000f3007c0100000000000000000000000000000000000000000000000000000000600035046347810fe381146100445780637e4a1aa81461005557806383d2421b1461006957005b61004f6004356100ab565b60006000f35b6100636004356024356100fc565b60006000f35b61007460043561007a565b60006000f35b6001543373ffffffffffffffffffffffffffffffffffffffff9081169116146100a2576100a8565b60078190555b50565b73ffffffffffffffffffffffffffffffffffffffff8116600090815260026020526040902080547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0016600117905550565b6001543373ffffffffffffffffffffffffffffffffffffffff9081169116146101245761012f565b600582905560068190555b505056"); Address address = Address.wrap(RandomUtils.nextBytes(Address.ADDRESS_LEN)); byte[] key_0 = ByteUtil.hexStringToBytes("18d63b70aa690ad37cb50908746c9a55"); @@ -197,48 +205,60 @@ public void test_2() throws Exception { AionContractDetailsImpl contractDetails_ = new AionContractDetailsImpl(data); - assertEquals(ByteUtil.toHexString(code), - ByteUtil.toHexString(contractDetails_.getCode())); + assertEquals(ByteUtil.toHexString(code), ByteUtil.toHexString(contractDetails_.getCode())); assertTrue(address.equals(contractDetails_.getAddress())); - assertEquals(ByteUtil.toHexString(val_1), + assertEquals( + ByteUtil.toHexString(val_1), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_1)).getData())); - assertEquals(ByteUtil.toHexString(val_2), + assertEquals( + ByteUtil.toHexString(val_2), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_2)).getData())); - assertEquals(ByteUtil.toHexString(val_3), + assertEquals( + ByteUtil.toHexString(val_3), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_3)).getData())); - assertEquals(ByteUtil.toHexString(val_4), + assertEquals( + ByteUtil.toHexString(val_4), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_4)).getData())); - assertEquals(ByteUtil.toHexString(val_5), + assertEquals( + ByteUtil.toHexString(val_5), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_5)).getData())); - assertEquals(ByteUtil.toHexString(val_6), + assertEquals( + ByteUtil.toHexString(val_6), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_6)).getData())); - assertEquals(ByteUtil.toHexString(val_7), + assertEquals( + ByteUtil.toHexString(val_7), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_7)).getData())); - assertEquals(ByteUtil.toHexString(val_8), + assertEquals( + ByteUtil.toHexString(val_8), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_8)).getData())); - assertEquals(ByteUtil.toHexString(val_9), + assertEquals( + ByteUtil.toHexString(val_9), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_9)).getData())); - assertEquals(ByteUtil.toHexString(val_10), + assertEquals( + ByteUtil.toHexString(val_10), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_10)).getData())); - assertEquals(ByteUtil.toHexString(val_11), + assertEquals( + ByteUtil.toHexString(val_11), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_11)).getData())); - assertEquals(ByteUtil.toHexString(val_12), + assertEquals( + ByteUtil.toHexString(val_12), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_12)).getData())); - assertEquals(ByteUtil.toHexString(val_13), + assertEquals( + ByteUtil.toHexString(val_13), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_13)).getData())); } @@ -310,7 +330,6 @@ public void testExternalStorageTransition() { original.put(key, value); } - original.syncStorage(); assertTrue(!externalStorage.isEmpty()); diff --git a/modAionImpl/test/org/aion/zero/impl/MockRepositoryConfig.java b/modAionImpl/test/org/aion/zero/impl/MockRepositoryConfig.java index 699260162a..15a29ea253 100644 --- a/modAionImpl/test/org/aion/zero/impl/MockRepositoryConfig.java +++ b/modAionImpl/test/org/aion/zero/impl/MockRepositoryConfig.java @@ -1,13 +1,14 @@ package org.aion.zero.impl; +import java.util.Properties; import org.aion.base.db.IContractDetails; +import org.aion.base.db.IPruneConfig; import org.aion.base.db.IRepositoryConfig; import org.aion.db.impl.DBVendor; import org.aion.db.impl.DatabaseFactory; +import org.aion.mcf.config.CfgPrune; import org.aion.zero.impl.db.ContractDetailsAion; -import java.util.Properties; - public class MockRepositoryConfig implements IRepositoryConfig { private DBVendor vendor = DBVendor.MOCKDB; @@ -17,8 +18,8 @@ public String getDbPath() { } @Override - public int getPrune() { - return 0; + public IPruneConfig getPruneConfig() { + return new CfgPrune(false); } @Override diff --git a/modAionImpl/test/org/aion/zero/impl/db/AionRepositoryImplTest.java b/modAionImpl/test/org/aion/zero/impl/db/AionRepositoryImplTest.java index 412e60b5b6..c05c59b414 100644 --- a/modAionImpl/test/org/aion/zero/impl/db/AionRepositoryImplTest.java +++ b/modAionImpl/test/org/aion/zero/impl/db/AionRepositoryImplTest.java @@ -39,15 +39,13 @@ import java.math.BigInteger; import java.util.Optional; import java.util.Properties; -import org.aion.base.db.IByteArrayKeyValueDatabase; -import org.aion.base.db.IContractDetails; -import org.aion.base.db.IRepositoryCache; -import org.aion.base.db.IRepositoryConfig; +import org.aion.base.db.*; import org.aion.base.type.Address; import org.aion.base.util.ByteUtil; import org.aion.crypto.HashUtil; import org.aion.db.impl.DBVendor; import org.aion.db.impl.DatabaseFactory; +import org.aion.mcf.config.CfgPrune; import org.aion.mcf.core.AccountState; import org.aion.mcf.db.IBlockStoreBase; import org.aion.mcf.vm.types.DataWord; @@ -67,8 +65,8 @@ public String getDbPath() { } @Override - public int getPrune() { - return 0; + public IPruneConfig getPruneConfig() { + return new CfgPrune(false); } @Override diff --git a/modBoot/resource/config.xml b/modBoot/resource/config.xml index 9b3a51e4e9..872285a788 100644 --- a/modBoot/resource/config.xml +++ b/modBoot/resource/config.xml @@ -64,8 +64,11 @@ database true - - 1000 + + + + + FULL leveldb diff --git a/modDbImpl/src/org/aion/db/generic/DatabaseWithCache.java b/modDbImpl/src/org/aion/db/generic/DatabaseWithCache.java index e2380e64ef..80636044ae 100644 --- a/modDbImpl/src/org/aion/db/generic/DatabaseWithCache.java +++ b/modDbImpl/src/org/aion/db/generic/DatabaseWithCache.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -163,14 +163,8 @@ public CacheStats getStats() { return this.loadingCache.stats(); } - /** - * Checks that the database connection is open. - * Throws a {@link RuntimeException} if the database connection is closed. - * - * @implNote Always do this check after acquiring a lock on the class/data. - * Otherwise it might produce inconsistent results due to lack of synchronization. - */ - private void check() { + @Override + public void check() { if (!database.isOpen()) { throw new RuntimeException("Database is not opened: " + this); } diff --git a/modDbImpl/src/org/aion/db/generic/LockedDatabase.java b/modDbImpl/src/org/aion/db/generic/LockedDatabase.java index ad7ed79c6d..9015430f35 100644 --- a/modDbImpl/src/org/aion/db/generic/LockedDatabase.java +++ b/modDbImpl/src/org/aion/db/generic/LockedDatabase.java @@ -366,6 +366,19 @@ public void deleteBatch(Collection keys) { } } + @Override + public void check() { + // acquire read lock + lock.readLock().lock(); + + try { + database.check(); + } finally { + // releasing read lock + lock.readLock().unlock(); + } + } + @Override public void drop() { // acquire write lock diff --git a/modDbImpl/src/org/aion/db/generic/TimedDatabase.java b/modDbImpl/src/org/aion/db/generic/TimedDatabase.java index 4a70fddc23..b101f6f697 100644 --- a/modDbImpl/src/org/aion/db/generic/TimedDatabase.java +++ b/modDbImpl/src/org/aion/db/generic/TimedDatabase.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -28,17 +28,16 @@ ******************************************************************************/ package org.aion.db.generic; +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.base.util.Hex; import org.aion.log.AionLoggerFactory; import org.aion.log.LogEnum; import org.slf4j.Logger; -import java.util.Collection; -import java.util.Map; -import java.util.Optional; -import java.util.Set; - /** * Times different database operations and logs the time. * @@ -48,6 +47,7 @@ public class TimedDatabase implements IByteArrayKeyValueDatabase { /** Unlocked database. */ protected final IByteArrayKeyValueDatabase database; + protected static final Logger LOG = AionLoggerFactory.getLogger(LogEnum.DB.name()); public TimedDatabase(IByteArrayKeyValueDatabase _database) { @@ -59,7 +59,8 @@ public String toString() { return this.getClass().getSimpleName() + " over " + database.toString(); } - // IDatabase functionality ----------------------------------------------------------------------------------------- + // IDatabase functionality + // ----------------------------------------------------------------------------------------- @Override public boolean open() { @@ -164,7 +165,8 @@ public long approximateSize() { return result; } - // IKeyValueStore functionality ------------------------------------------------------------------------------------ + // IKeyValueStore functionality + // ------------------------------------------------------------------------------------ @Override public boolean isEmpty() { @@ -192,7 +194,13 @@ public Optional get(byte[] key) { Optional value = database.get(key); long t2 = System.nanoTime(); - LOG.debug(database.toString() + " get(key) in " + (t2 - t1) + " ns." + "\n\t\t\t\t\tkey = " + Hex.toHexString(key)); + LOG.debug( + database.toString() + + " get(key) in " + + (t2 - t1) + + " ns." + + "\n\t\t\t\t\tkey = " + + (key != null ? Hex.toHexString(key) : "null")); return value; } @@ -202,8 +210,15 @@ public void put(byte[] key, byte[] value) { database.put(key, value); long t2 = System.nanoTime(); - LOG.debug(database.toString() + " put(key,value) in " + (t2 - t1) + " ns." + "\n\t\t\t\t\tkey = " + Hex.toHexString(key) - + "\n\t\t\t\t\tvalue = " + Hex.toHexString(value)); + LOG.debug( + database.toString() + + " put(key,value) in " + + (t2 - t1) + + " ns." + + "\n\t\t\t\t\tkey = " + + (key != null ? Hex.toHexString(key) : "null") + + "\n\t\t\t\t\tvalue = " + + (value != null ? Hex.toHexString(value) : "null")); } @Override @@ -212,7 +227,13 @@ public void delete(byte[] key) { database.delete(key); long t2 = System.nanoTime(); - LOG.debug(database.toString() + " delete(key) in " + (t2 - t1) + " ns." + "\n\t\t\t\t\tkey = " + Hex.toHexString(key)); + LOG.debug( + database.toString() + + " delete(key) in " + + (t2 - t1) + + " ns." + + "\n\t\t\t\t\tkey = " + + (key != null ? Hex.toHexString(key) : "null")); } @Override @@ -221,7 +242,13 @@ public void putBatch(Map keyValuePairs) { database.putBatch(keyValuePairs); long t2 = System.nanoTime(); - LOG.debug(database.toString() + " putBatch(" + keyValuePairs.size() + ") in " + (t2 - t1) + " ns."); + LOG.debug( + database.toString() + + " putBatch(" + + (keyValuePairs != null ? keyValuePairs.size() : "null") + + ") in " + + (t2 - t1) + + " ns."); } @Override @@ -230,8 +257,15 @@ public void putToBatch(byte[] key, byte[] value) { database.putToBatch(key, value); long t2 = System.nanoTime(); - LOG.debug(database.toString() + " putToBatch(key,value) in " + (t2 - t1) + " ns." + "\n\t\t\t\t\tkey = " + Hex - .toHexString(key) + "\n\t\t\t\t\tvalue = " + Hex.toHexString(value)); + LOG.debug( + database.toString() + + " putToBatch(key,value) in " + + (t2 - t1) + + " ns." + + "\n\t\t\t\t\tkey = " + + Hex.toHexString(key) + + "\n\t\t\t\t\tvalue = " + + (value != null ? Hex.toHexString(value) : "null")); } @Override @@ -249,7 +283,22 @@ public void deleteBatch(Collection keys) { database.deleteBatch(keys); long t2 = System.nanoTime(); - LOG.debug(database.toString() + " deleteBatch(" + keys.size() + ") in " + (t2 - t1) + " ns."); + LOG.debug( + database.toString() + + " deleteBatch(" + + (keys != null ? keys.size() : "null") + + ") in " + + (t2 - t1) + + " ns."); + } + + @Override + public void check() { + long t1 = System.nanoTime(); + database.check(); + long t2 = System.nanoTime(); + + LOG.debug(database.toString() + " check() in " + (t2 - t1) + " ns."); } @Override diff --git a/modDbImpl/src/org/aion/db/impl/AbstractDB.java b/modDbImpl/src/org/aion/db/impl/AbstractDB.java index ee346f2ec8..9fa6243eed 100644 --- a/modDbImpl/src/org/aion/db/impl/AbstractDB.java +++ b/modDbImpl/src/org/aion/db/impl/AbstractDB.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -34,18 +34,16 @@ ******************************************************************************/ package org.aion.db.impl; -import org.aion.base.db.IByteArrayKeyValueDatabase; -import org.aion.base.util.ByteArrayWrapper; -import org.aion.log.AionLoggerFactory; -import org.aion.log.LogEnum; -import org.h2.store.fs.FileUtils; -import org.slf4j.Logger; - import java.io.File; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; import java.util.stream.Stream; +import org.aion.base.db.IByteArrayKeyValueDatabase; +import org.aion.base.util.ByteArrayWrapper; +import org.aion.log.AionLoggerFactory; +import org.aion.log.LogEnum; +import org.slf4j.Logger; /** * Common functionality for database implementations. @@ -70,7 +68,8 @@ protected AbstractDB(String name) { this.name = name; } - protected AbstractDB(String name, String path, boolean enableDbCache, boolean enableDbCompression) { + protected AbstractDB( + String name, String path, boolean enableDbCache, boolean enableDbCompression) { this(name); Objects.requireNonNull(path, "The database path cannot be null."); @@ -81,14 +80,22 @@ protected AbstractDB(String name, String path, boolean enableDbCache, boolean en } protected String propertiesInfo() { - return ""; // + return ""; // } @Override public boolean commit() { - // not implemented since we always commit the changes to the database for this implementation - throw new UnsupportedOperationException("Only automatic commits are supported by " + this.toString()); + // not implemented since we always commit the changes to the database for this + // implementation + throw new UnsupportedOperationException( + "Only automatic commits are supported by " + this.toString()); } @Override @@ -119,22 +126,16 @@ public Optional getPath() { return Optional.ofNullable(this.path); } - /** - * Checks that the database connection is open. - * Throws a {@link RuntimeException} if the database connection is closed. - * - * @implNote Always do this check after acquiring a lock on the class/data. - * Otherwise it might produce inconsistent results due to lack of synchronization. - */ - protected void check() { + @Override + public void check() { if (!isOpen()) { throw new RuntimeException("Database is not opened: " + this); } } /** - * Checks that the given key is not null. - * Throws a {@link IllegalArgumentException} if the key is null. + * Checks that the given key is not null. Throws a {@link IllegalArgumentException} if the key + * is null. */ public static void check(byte[] k) { if (k == null) { @@ -143,8 +144,8 @@ public static void check(byte[] k) { } /** - * Checks that the given collection of keys does not contain null values. - * Throws a {@link IllegalArgumentException} if a null key is present. + * Checks that the given collection of keys does not contain null values. Throws a {@link + * IllegalArgumentException} if a null key is present. */ public static void check(Collection keys) { if (keys.contains(null)) { @@ -170,23 +171,21 @@ public boolean isPersistent() { } /** - * For testing the lock functionality of public methods. - * Helps ensure that locks are released after normal or exceptional execution. + * For testing the lock functionality of public methods. Helps ensure that locks are released + * after normal or exceptional execution. * - * @return {@code true} when the resource is locked, - * {@code false} otherwise + * @return {@code true} when the resource is locked, {@code false} otherwise */ @Override public boolean isLocked() { return false; } - /** - * Functionality for directly interacting with the heap cache. - */ + /** Functionality for directly interacting with the heap cache. */ public abstract boolean commitCache(Map cache); - // IKeyValueStore functionality ------------------------------------------------------------------------------------ + // IKeyValueStore functionality + // ------------------------------------------------------------------------------------ @Override public Optional get(byte[] k) { @@ -200,12 +199,11 @@ public Optional get(byte[] k) { } /** - * Database specific get functionality, without locking required. Locking is applied in {@link #get(byte[])}. + * Database specific get functionality, without locking required. Locking is applied in {@link + * #get(byte[])}. * - * @param k - * the key for which the method must return the associated value + * @param k the key for which the method must return the associated value * @return the value stored in the database for the give key. */ protected abstract byte[] getInternal(byte[] k); - -} \ No newline at end of file +} diff --git a/modMcf/src/org/aion/mcf/config/CfgDb.java b/modMcf/src/org/aion/mcf/config/CfgDb.java index 9f39813acd..ae79157455 100644 --- a/modMcf/src/org/aion/mcf/config/CfgDb.java +++ b/modMcf/src/org/aion/mcf/config/CfgDb.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -22,25 +22,22 @@ ******************************************************************************/ package org.aion.mcf.config; -import org.aion.base.util.Utils; -import org.aion.db.impl.DBVendor; +import static org.aion.db.impl.DatabaseFactory.Props; -import javax.xml.stream.XMLOutputFactory; -import javax.xml.stream.XMLStreamException; -import javax.xml.stream.XMLStreamReader; -import javax.xml.stream.XMLStreamWriter; import java.io.IOException; import java.io.StringWriter; import java.io.Writer; import java.util.HashMap; import java.util.Map; import java.util.Properties; +import javax.xml.stream.XMLOutputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import javax.xml.stream.XMLStreamWriter; +import org.aion.base.util.Utils; +import org.aion.db.impl.DBVendor; -import static org.aion.db.impl.DatabaseFactory.Props; - -/** - * @author chris - */ +/** @author chris */ public class CfgDb { public static class Names { @@ -53,6 +50,7 @@ public static class Names { public static final String STORAGE = "storage"; public static final String STATE = "state"; + public static final String STATE_ARCHIVE = "stateArchive"; public static final String TRANSACTION = "transaction"; public static final String TX_CACHE = "pendingtxCache"; @@ -63,7 +61,8 @@ public static class Names { private String vendor; private boolean compression; private boolean check_integrity; - private int prune; + private CfgPrune prune; + private PruneOption prune_option; /** * Enabling expert mode allows more detailed database configurations. @@ -80,7 +79,8 @@ public CfgDb() { this.vendor = DBVendor.LEVELDB.toValue(); this.compression = false; this.check_integrity = true; - this.prune = -1; + this.prune = new CfgPrune(false); + this.prune_option = PruneOption.FULL; if (expert) { this.specificConfig = new HashMap<>(); @@ -102,80 +102,89 @@ public void fromXML(final XMLStreamReader sr) throws XMLStreamException { case "check_integrity": this.check_integrity = Boolean.parseBoolean(Cfg.readValue(sr)); break; - case "prune": - this.prune = Integer.parseInt(Cfg.readValue(sr)); + case "state-storage": + setPrune(Cfg.readValue(sr)); break; - // parameter considered only when expert==false + // parameter considered only when expert==false case "vendor": this.vendor = Cfg.readValue(sr); break; - // parameter considered only when expert==false + // parameter considered only when expert==false case Props.ENABLE_DB_COMPRESSION: this.compression = Boolean.parseBoolean(Cfg.readValue(sr)); break; - // parameter considered only when expert==true - case Names.DEFAULT: { - CfgDbDetails dbConfig = this.specificConfig.get(Names.DEFAULT); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.DEFAULT, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.BLOCK: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.BLOCK, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.INDEX: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.INDEX, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.DETAILS: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.DETAILS, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.STORAGE: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.STORAGE, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.STATE: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.STATE, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.TRANSACTION: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.TRANSACTION, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.TX_POOL: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.TX_POOL, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.TX_CACHE: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.TX_CACHE, dbConfig); - break; - } + // parameter considered only when expert==true + case Names.DEFAULT: + { + CfgDbDetails dbConfig = this.specificConfig.get(Names.DEFAULT); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.DEFAULT, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.BLOCK: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.BLOCK, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.INDEX: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.INDEX, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.DETAILS: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.DETAILS, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.STORAGE: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.STORAGE, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.STATE: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.STATE, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.TRANSACTION: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.TRANSACTION, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.TX_POOL: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.TX_POOL, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.TX_CACHE: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.TX_CACHE, dbConfig); + break; + } default: Cfg.skipElement(sr); break; @@ -205,17 +214,27 @@ public String toXML() { xmlWriter.writeEndElement(); xmlWriter.writeCharacters("\r\n\t\t"); - xmlWriter.writeComment("Boolean value. Enable/disable database integrity check run at startup."); + xmlWriter.writeComment( + "Boolean value. Enable/disable database integrity check run at startup."); xmlWriter.writeCharacters("\r\n\t\t"); xmlWriter.writeStartElement("check_integrity"); xmlWriter.writeCharacters(String.valueOf(this.check_integrity)); xmlWriter.writeEndElement(); xmlWriter.writeCharacters("\r\n\t\t"); - xmlWriter.writeComment("Integer value. Number of blocks after which to prune. Pruning disabled when negative."); + xmlWriter.writeComment( + "Data pruning behavior for the state database. Options: FULL, TOP, SPREAD."); xmlWriter.writeCharacters("\r\n\t\t"); - xmlWriter.writeStartElement("prune"); - xmlWriter.writeCharacters(String.valueOf(this.prune)); + xmlWriter.writeComment("FULL: the state is not pruned"); + xmlWriter.writeCharacters("\r\n\t\t"); + xmlWriter.writeComment( + "TOP: the state is kept only for the top K blocks; limits sync to branching only within the stored blocks"); + xmlWriter.writeCharacters("\r\n\t\t"); + xmlWriter.writeComment( + "SPREAD: the state is kept for the top K blocks and at regular block intervals"); + xmlWriter.writeCharacters("\r\n\t\t"); + xmlWriter.writeStartElement("state-storage"); + xmlWriter.writeCharacters(this.prune_option.toString()); xmlWriter.writeEndElement(); if (!expert) { @@ -223,7 +242,8 @@ public String toXML() { xmlWriter.writeComment( "Database implementation used to store data; supported options: leveldb, h2, rocksdb."); xmlWriter.writeCharacters("\r\n\t\t"); - xmlWriter.writeComment("Caution: changing implementation requires re-syncing from genesis!"); + xmlWriter.writeComment( + "Caution: changing implementation requires re-syncing from genesis!"); xmlWriter.writeCharacters("\r\n\t\t"); xmlWriter.writeStartElement("vendor"); xmlWriter.writeCharacters(this.vendor); @@ -260,10 +280,73 @@ public String getPath() { return this.path; } - public int getPrune() { + public CfgPrune getPrune() { return this.prune; } + /** + * Number of topmost blocks present in the database in TOP pruning mode. Information about these + * blocks is also kept in memory for later pruning. + */ + public static final int TOP_PRUNE_BLOCK_COUNT = 256; + /** + * Number of topmost blocks present in the database in SPREAD pruning mode. Information about + * these blocks is also kept in memory for later pruning. + */ + public static final int SPREAD_PRUNE_BLOCK_COUNT = 128; + /** At what frequency block states are being archived. */ + public static final int SPREAD_PRUNE_ARCHIVE_RATE = 10000; + + public enum PruneOption { + FULL, + TOP, + SPREAD; + + @Override + public String toString() { + return this.name(); + } + + public static PruneOption fromValue(String value) { + value = value.toUpperCase(); + + if (value != null) { + for (PruneOption color : values()) { + if (color.toString().equals(value)) { + return color; + } + } + } + + // return default value + return getDefault(); + } + + public static PruneOption getDefault() { + return FULL; + } + } + + public void setPrune(String _prune_option) { + this.prune_option = PruneOption.fromValue(_prune_option); + + switch (prune_option) { + case TOP: + // journal prune only + this.prune = new CfgPrune(TOP_PRUNE_BLOCK_COUNT); + break; + case SPREAD: + // journal prune with archived states + this.prune = new CfgPrune(SPREAD_PRUNE_BLOCK_COUNT, SPREAD_PRUNE_ARCHIVE_RATE); + break; + case FULL: + default: + // the default is no pruning + this.prune = new CfgPrune(false); + break; + } + } + public Map asProperties() { Map propSet = new HashMap<>(); @@ -307,4 +390,4 @@ public void setHeapCacheEnabled(boolean value) { } } } -} \ No newline at end of file +} diff --git a/modMcf/src/org/aion/mcf/config/CfgPrune.java b/modMcf/src/org/aion/mcf/config/CfgPrune.java new file mode 100644 index 0000000000..842c483fe2 --- /dev/null +++ b/modMcf/src/org/aion/mcf/config/CfgPrune.java @@ -0,0 +1,172 @@ +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: + * Aion foundation. + ******************************************************************************/ +package org.aion.mcf.config; + +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import javax.xml.stream.XMLStreamWriter; +import org.aion.base.db.IPruneConfig; + +/** + * Configuration for data pruning behavior. + * + * @author Alexandra Roatis + */ +public class CfgPrune implements IPruneConfig { + + private boolean enabled; + private boolean archived; + private int current_count = MINIMUM_CURRENT_COUNT; + private int archive_rate = MINIMUM_ARCHIVE_RATE; + + private static final int MINIMUM_CURRENT_COUNT = 128; + private static final int MINIMUM_ARCHIVE_RATE = 1000; + + public CfgPrune(boolean _enabled) { + this.enabled = _enabled; + this.archived = _enabled; + } + + public CfgPrune(int _current_count) { + // enable journal pruning + this.enabled = true; + this.current_count = + _current_count > MINIMUM_CURRENT_COUNT ? _current_count : MINIMUM_CURRENT_COUNT; + // disable archiving + this.archived = false; + } + + public CfgPrune(int _current_count, int _archive_rate) { + // enable journal pruning + this.enabled = true; + this.current_count = + _current_count > MINIMUM_CURRENT_COUNT ? _current_count : MINIMUM_CURRENT_COUNT; + // enable archiving + this.archived = true; + this.archive_rate = + _archive_rate > MINIMUM_ARCHIVE_RATE ? _archive_rate : MINIMUM_ARCHIVE_RATE; + } + + public void fromXML(final XMLStreamReader sr) throws XMLStreamException { + loop: + while (sr.hasNext()) { + int eventType = sr.next(); + switch (eventType) { + case XMLStreamReader.START_ELEMENT: + String elementName = sr.getLocalName().toLowerCase(); + switch (elementName) { + case "enabled": + this.enabled = Boolean.parseBoolean(Cfg.readValue(sr)); + break; + case "archived": + this.archived = Boolean.parseBoolean(Cfg.readValue(sr)); + break; + case "current_count": + this.current_count = Integer.parseInt(Cfg.readValue(sr)); + // must be at least MINIMUM_CURRENT_COUNT + if (this.current_count < MINIMUM_CURRENT_COUNT) { + this.current_count = MINIMUM_CURRENT_COUNT; + } + break; + case "archive_rate": + this.archive_rate = Integer.parseInt(Cfg.readValue(sr)); + // must be at least MINIMUM_ARCHIVE_RATE + if (this.archive_rate < MINIMUM_ARCHIVE_RATE) { + this.archive_rate = MINIMUM_ARCHIVE_RATE; + } + break; + default: + Cfg.skipElement(sr); + break; + } + break; + case XMLStreamReader.END_ELEMENT: + break loop; + } + } + } + + public void toXML(XMLStreamWriter xmlWriter) throws XMLStreamException { + xmlWriter.writeCharacters("\r\n\t\t"); + xmlWriter.writeStartElement("prune"); + + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeComment("Boolean value. Enable/disable database pruning."); + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeStartElement("enabled"); + xmlWriter.writeCharacters(String.valueOf(this.enabled)); + xmlWriter.writeEndElement(); + + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeComment("Boolean value. Enable/disable database archiving."); + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeStartElement("archived"); + xmlWriter.writeCharacters(String.valueOf(this.archived)); + xmlWriter.writeEndElement(); + + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeComment( + "Integer value with minimum set to 128. Only blocks older than best block level minus this number are candidates for pruning."); + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeStartElement("current_count"); + xmlWriter.writeCharacters(String.valueOf(this.current_count)); + xmlWriter.writeEndElement(); + + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeComment( + "Integer value with minimum set to 1000. States for blocks that are exact multiples of this number will not be pruned."); + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeStartElement("archive_rate"); + xmlWriter.writeCharacters(String.valueOf(this.archive_rate)); + xmlWriter.writeEndElement(); + + xmlWriter.writeCharacters("\r\n\t\t"); + xmlWriter.writeEndElement(); + } + + @Override + public boolean isEnabled() { + return enabled; + } + + @Override + public boolean isArchived() { + return archived; + } + + @Override + public int getCurrentCount() { + return current_count; + } + + @Override + public int getArchiveRate() { + return archive_rate; + } +} diff --git a/modMcf/src/org/aion/mcf/db/AbstractRepository.java b/modMcf/src/org/aion/mcf/db/AbstractRepository.java index f894acd753..60efd98d76 100644 --- a/modMcf/src/org/aion/mcf/db/AbstractRepository.java +++ b/modMcf/src/org/aion/mcf/db/AbstractRepository.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * * Copyright (c) 2017, 2018 Aion foundation. * @@ -18,9 +18,17 @@ * Contributors: * Aion foundation. *******************************************************************************/ - package org.aion.mcf.db; +import static org.aion.db.impl.DatabaseFactory.Props; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Objects; +import java.util.Properties; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.base.db.IRepository; import org.aion.base.db.IRepositoryConfig; @@ -32,28 +40,20 @@ import org.aion.mcf.config.CfgDb; import org.aion.mcf.core.AccountState; import org.aion.mcf.db.exception.InvalidFilePathException; +import org.aion.mcf.ds.ArchivedDataSource; import org.aion.mcf.trie.JournalPruneDataSource; import org.aion.mcf.trie.Trie; import org.aion.mcf.types.AbstractBlock; import org.aion.mcf.vm.types.DataWord; import org.slf4j.Logger; -import java.io.File; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Objects; -import java.util.Properties; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import static org.aion.db.impl.DatabaseFactory.Props; - -//import org.aion.dbmgr.exception.DriverManagerNoSuitableDriverRegisteredException; +// import org.aion.dbmgr.exception.DriverManagerNoSuitableDriverRegisteredException; -/** - * Abstract Repository class. - */ -public abstract class AbstractRepository, BH extends IBlockHeader, BSB extends IBlockStoreBase> +/** Abstract Repository class. */ +public abstract class AbstractRepository< + BLK extends AbstractBlock, + BH extends IBlockHeader, + BSB extends IBlockStoreBase> implements IRepository { // Logger @@ -63,14 +63,15 @@ public abstract class AbstractRepository databaseGroup; - protected JournalPruneDataSource stateDSPrune; + protected ArchivedDataSource stateWithArchive; + protected JournalPruneDataSource stateDSPrune; protected DetailsDataStore detailsDS; // Read Write Lock @@ -103,6 +106,7 @@ public abstract class AbstractRepository vendorListString = new ArrayList<>(); -// for (String v : this.cfg.getVendorList()) { -// vendorListString.add("\"" + v + "\""); -// } -// throw new DriverManagerNoSuitableDriverRegisteredException( -// "Please check the vendor name field in /config/config.xml.\n" -// + "No suitable driver found with name \"" + this.cfg.getActiveVendor() -// + "\".\nPlease select a driver from the following vendor list: " + vendorListString); -// } + // } + // + // if (!Arrays.asList(this.cfg.getVendorList()).contains(this.cfg.getActiveVendor())) + // { + // + // ArrayList vendorListString = new ArrayList<>(); + // for (String v : this.cfg.getVendorList()) { + // vendorListString.add("\"" + v + "\""); + // } + // throw new DriverManagerNoSuitableDriverRegisteredException( + // "Please check the vendor name field in /config/config.xml.\n" + // + "No suitable driver found with name \"" + + // this.cfg.getActiveVendor() + // + "\".\nPlease select a driver from the following vendor list: + // " + vendorListString); + // } Properties sharedProps; @@ -171,13 +180,15 @@ protected void initializeDatabasesAndCaches() throws Exception { try { databaseGroup = new ArrayList<>(); - checkIntegrity = Boolean - .valueOf(cfg.getDatabaseConfig(CfgDb.Names.DEFAULT).getProperty(Props.CHECK_INTEGRITY)); + checkIntegrity = + Boolean.valueOf( + cfg.getDatabaseConfig(CfgDb.Names.DEFAULT) + .getProperty(Props.CHECK_INTEGRITY)); // getting state specific properties sharedProps = cfg.getDatabaseConfig(STATE_DB); - // locking enabled for state - sharedProps.setProperty(Props.ENABLE_LOCKING, "true"); + // locking enabled for state when JournalPrune not used + sharedProps.setProperty(Props.ENABLE_LOCKING, "false"); sharedProps.setProperty(Props.DB_PATH, cfg.getDbPath()); sharedProps.setProperty(Props.DB_NAME, STATE_DB); this.stateDatabase = connectAndOpen(sharedProps); @@ -241,13 +252,39 @@ protected void initializeDatabasesAndCaches() throws Exception { // Setup the cache for transaction data source. this.detailsDS = new DetailsDataStore<>(detailsDatabase, storageDatabase, this.cfg); - stateDSPrune = new JournalPruneDataSource<>(stateDatabase); - pruneBlockCount = pruneEnabled ? this.cfg.getPrune() : -1; - if (pruneEnabled && pruneBlockCount > 0) { - LOGGEN.info("Pruning block count set to {}.", pruneBlockCount); + + // pruning config + pruneEnabled = this.cfg.getPruneConfig().isEnabled(); + pruneBlockCount = this.cfg.getPruneConfig().getCurrentCount(); + archiveRate = this.cfg.getPruneConfig().getArchiveRate(); + + if (pruneEnabled && this.cfg.getPruneConfig().isArchived()) { + // using state config for state_archive + sharedProps = cfg.getDatabaseConfig(STATE_DB); + sharedProps.setProperty(Props.ENABLE_LOCKING, "false"); + sharedProps.setProperty(Props.DB_PATH, cfg.getDbPath()); + sharedProps.setProperty(Props.DB_NAME, STATE_ARCHIVE_DB); + this.stateArchiveDatabase = connectAndOpen(sharedProps); + databaseGroup.add(stateArchiveDatabase); + + stateWithArchive = new ArchivedDataSource(stateDatabase, stateArchiveDatabase); + stateDSPrune = new JournalPruneDataSource(stateWithArchive); + + LOGGEN.info( + "Pruning and archiving ENABLED. Top block count set to {} and archive rate set to {}.", + pruneBlockCount, + archiveRate); } else { - stateDSPrune.setPruneEnabled(false); + stateArchiveDatabase = null; + stateWithArchive = null; + stateDSPrune = new JournalPruneDataSource(stateDatabase); + + if (pruneEnabled) { + LOGGEN.info("Pruning ENABLED. Top block count set to {}.", pruneBlockCount); + } } + + stateDSPrune.setPruneEnabled(pruneEnabled); } catch (Exception e) { // Setting up databases and caches went wrong. throw e; } @@ -272,16 +309,18 @@ private IByteArrayKeyValueDatabase connectAndOpen(Properties info) { // check object status if (db == null) { - LOG.error("Database <{}> connection could not be established for <{}>.", - info.getProperty(Props.DB_TYPE), - info.getProperty(Props.DB_NAME)); + LOG.error( + "Database <{}> connection could not be established for <{}>.", + info.getProperty(Props.DB_TYPE), + info.getProperty(Props.DB_NAME)); } // check persistence status if (!db.isCreatedOnDisk()) { - LOG.error("Database <{}> cannot be saved to disk for <{}>.", - info.getProperty(Props.DB_TYPE), - info.getProperty(Props.DB_NAME)); + LOG.error( + "Database <{}> cannot be saved to disk for <{}>.", + info.getProperty(Props.DB_TYPE), + info.getProperty(Props.DB_NAME)); } return db; diff --git a/modMcf/src/org/aion/mcf/db/DetailsDataStore.java b/modMcf/src/org/aion/mcf/db/DetailsDataStore.java index c97997f95f..280cd38787 100644 --- a/modMcf/src/org/aion/mcf/db/DetailsDataStore.java +++ b/modMcf/src/org/aion/mcf/db/DetailsDataStore.java @@ -1,25 +1,42 @@ -/******************************************************************************* +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. * - * Copyright (c) 2017, 2018 Aion foundation. + * This file is part of the aion network project. * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see + * along with the aion network project source files. + * If not, see . * - * Contributors: + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: * Aion foundation. - *******************************************************************************/ + * team through the ethereumJ library. + * Ether.Camp Inc. (US) team through Ethereum Harmony. + * John Tromp through the Equihash solver. + * Samuel Neves through the BLAKE2 implementation. + * Zcash project team. + * Bitcoinj team. + ******************************************************************************/ package org.aion.mcf.db; +import static org.aion.base.util.ByteArrayWrapper.wrap; + +import java.util.*; import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.base.db.IContractDetails; import org.aion.base.db.IRepositoryConfig; @@ -31,43 +48,38 @@ import org.aion.mcf.types.AbstractBlock; import org.aion.mcf.vm.types.DataWord; -import java.util.*; - -import static org.aion.base.util.ByteArrayWrapper.wrap; - -/** - * Detail data storage , - */ -public class DetailsDataStore, BH extends IBlockHeader> { +/** Detail data storage , */ +public class DetailsDataStore< + BLK extends AbstractBlock, BH extends IBlockHeader> { - private JournalPruneDataSource storageDSPrune; + private JournalPruneDataSource storageDSPrune; private IRepositoryConfig repoConfig; private IByteArrayKeyValueDatabase detailsSrc; private IByteArrayKeyValueDatabase storageSrc; private Set removes = new HashSet<>(); - public DetailsDataStore() { - } + public DetailsDataStore() {} - public DetailsDataStore(IByteArrayKeyValueDatabase detailsCache, IByteArrayKeyValueDatabase storageCache, + public DetailsDataStore( + IByteArrayKeyValueDatabase detailsCache, + IByteArrayKeyValueDatabase storageCache, IRepositoryConfig repoConfig) { this.repoConfig = repoConfig; withDb(detailsCache, storageCache); } - public DetailsDataStore withDb(IByteArrayKeyValueDatabase detailsSrc, - IByteArrayKeyValueDatabase storageSrc) { + public DetailsDataStore withDb( + IByteArrayKeyValueDatabase detailsSrc, IByteArrayKeyValueDatabase storageSrc) { this.detailsSrc = detailsSrc; this.storageSrc = storageSrc; - this.storageDSPrune = new JournalPruneDataSource<>(storageSrc); + this.storageDSPrune = new JournalPruneDataSource(storageSrc); return this; } /** - * Fetches the ContractDetails from the cache, and if it doesn't exist, add - * to the remove set. + * Fetches the ContractDetails from the cache, and if it doesn't exist, add to the remove set. * * @param key * @return @@ -110,7 +122,6 @@ public synchronized void update(Address key, IContractDetails contract // Remove from the remove set. removes.remove(wrappedKey); - } public synchronized void remove(byte[] key) { @@ -170,7 +181,7 @@ public void syncLargeStorage() { } } - public JournalPruneDataSource getStorageDSPrune() { + public JournalPruneDataSource getStorageDSPrune() { return storageDSPrune; } diff --git a/modMcf/src/org/aion/mcf/ds/ArchivedDataSource.java b/modMcf/src/org/aion/mcf/ds/ArchivedDataSource.java new file mode 100644 index 0000000000..0f0f35d2fa --- /dev/null +++ b/modMcf/src/org/aion/mcf/ds/ArchivedDataSource.java @@ -0,0 +1,135 @@ +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: + * Aion foundation. + ******************************************************************************/ +package org.aion.mcf.ds; + +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import org.aion.base.db.IByteArrayKeyValueDatabase; +import org.aion.base.db.IByteArrayKeyValueStore; + +/** + * A data source with archived data that must no be deleted. + * + * @author Alexandra Roatis + */ +public class ArchivedDataSource implements IByteArrayKeyValueStore { + + IByteArrayKeyValueDatabase data, archive; + + public ArchivedDataSource(IByteArrayKeyValueDatabase _db, IByteArrayKeyValueDatabase _archive) { + this.data = _db; + this.archive = _archive; + } + + @Override + public boolean isEmpty() { + return data.isEmpty(); + } + + @Override + public Set keys() { + return data.keys(); + } + + @Override + public Optional get(byte[] key) { + return data.get(key); + } + + @Override + public void put(byte[] key, byte[] value) { + if (value != null) { + data.put(key, value); + } else { + // internal delete will check if archived + delete(key); + } + } + + @Override + public void delete(byte[] key) { + // delete key only if not archived + if (!archive.get(key).isPresent()) { + data.delete(key); + } + } + + @Override + public void putBatch(Map batch) { + for (Map.Entry entry : batch.entrySet()) { + // will check if archived + putToBatch(entry.getKey(), entry.getValue()); + } + commitBatch(); + } + + @Override + public void putToBatch(byte[] key, byte[] value) { + if (value != null) { + data.putToBatch(key, value); + } else { + // deleted key only if not archived + if (!archive.get(key).isPresent()) { + data.putToBatch(key, null); + } + } + } + + @Override + public void commitBatch() { + data.commitBatch(); + } + + @Override + public void deleteBatch(Collection keys) { + for (byte[] key : keys) { + // will check if archived + putToBatch(key, null); + } + commitBatch(); + } + + @Override + public void check() { + data.check(); + archive.check(); + } + + @Override + public void close() { + data.close(); + archive.close(); + } + + public IByteArrayKeyValueDatabase getArchiveDatabase() { + return archive; + } +} diff --git a/modMcf/src/org/aion/mcf/ds/XorDataSource.java b/modMcf/src/org/aion/mcf/ds/XorDataSource.java index e4f77cbe52..f71259481d 100644 --- a/modMcf/src/org/aion/mcf/ds/XorDataSource.java +++ b/modMcf/src/org/aion/mcf/ds/XorDataSource.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -34,13 +34,11 @@ ******************************************************************************/ package org.aion.mcf.ds; -import org.aion.base.db.IByteArrayKeyValueDatabase; +import java.util.*; import org.aion.base.db.IByteArrayKeyValueStore; import org.aion.base.util.ByteArrayWrapper; import org.aion.base.util.ByteUtil; -import java.util.*; - public class XorDataSource implements IByteArrayKeyValueStore { IByteArrayKeyValueStore source; byte[] subKey; @@ -89,8 +87,9 @@ public void putBatch(Map rows) { } public void updateBatch(Map rows, boolean erasure) { - //not supported - throw new UnsupportedOperationException("ByteArrayWrapper map not supported in XorDataSource.updateBatch yet"); + // not supported + throw new UnsupportedOperationException( + "ByteArrayWrapper map not supported in XorDataSource.updateBatch yet"); } @Override @@ -104,6 +103,11 @@ public void deleteBatch(Collection keys) { } + @Override + public void check() { + source.check(); + } + @Override public boolean isEmpty() { // TODO Auto-generated method stub diff --git a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java index f547901dfd..05aca77569 100644 --- a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java +++ b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java @@ -35,11 +35,16 @@ package org.aion.mcf.trie; import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.base.db.IByteArrayKeyValueStore; -import org.aion.base.type.IBlock; -import org.aion.base.type.IBlockHeader; import org.aion.base.util.ByteArrayWrapper; +import org.aion.log.AionLoggerFactory; +import org.aion.log.LogEnum; +import org.aion.mcf.ds.ArchivedDataSource; +import org.slf4j.Logger; /** * The DataSource which doesn't immediately forward delete updates (unlike inserts) but collects @@ -48,12 +53,14 @@ * submitted to the underlying DataSource with respect to following inserts. E.g. if the key was * deleted at block N and then inserted at block N + 10 this delete is not passed. */ -public class JournalPruneDataSource, BH extends IBlockHeader> - implements IByteArrayKeyValueStore { +public class JournalPruneDataSource implements IByteArrayKeyValueStore { - private class Updates { + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private static final Logger LOG = AionLoggerFactory.getLogger(LogEnum.DB.name()); - BH blockHeader; + private class Updates { + ByteArrayWrapper blockHeader; + long blockNumber; Set insertedKeys = new HashSet<>(); Set deletedKeys = new HashSet<>(); } @@ -70,79 +77,146 @@ public Ref(boolean dbRef) { public int getTotRefs() { return journalRefs + (dbRef ? 1 : 0); } + + @Override + public String toString() { + return "refs: " + String.valueOf(journalRefs) + " db: " + String.valueOf(dbRef); + } } Map refCount = new HashMap<>(); - private IByteArrayKeyValueDatabase src; + private IByteArrayKeyValueStore src; // block hash => updates private LinkedHashMap blockUpdates = new LinkedHashMap<>(); private Updates currentUpdates = new Updates(); - private boolean enabled = true; + private AtomicBoolean enabled = new AtomicBoolean(false); + private final boolean hasArchive; - public JournalPruneDataSource(IByteArrayKeyValueDatabase src) { + public JournalPruneDataSource(IByteArrayKeyValueStore src) { this.src = src; + this.hasArchive = src instanceof ArchivedDataSource; + } + + public void setPruneEnabled(boolean _enabled) { + enabled.set(_enabled); } - public void setPruneEnabled(boolean e) { - enabled = e; + public boolean isArchiveEnabled() { + return hasArchive; } - public synchronized void put(byte[] key, byte[] value) { - ByteArrayWrapper keyW = new ByteArrayWrapper(key); + public void put(byte[] key, byte[] value) { + checkNotNull(key); - // Check to see the value exists. - if (value != null) { + lock.writeLock().lock(); - // If it exists and pruning is enabled. - if (enabled) { - currentUpdates.insertedKeys.add(keyW); - incRef(keyW); - } + try { + if (enabled.get()) { + // pruning enabled + ByteArrayWrapper keyW = ByteArrayWrapper.wrap(key); - // put to source database. - src.put(key, value); + // Check to see the value exists. + if (value != null) { + // If it exists and pruning is enabled. + currentUpdates.insertedKeys.add(keyW); + incRef(keyW); - } else { - // Value does not exist, so we delete from current updates - if (enabled) { - currentUpdates.deletedKeys.add(keyW); + // put to source database. + src.put(key, value); + + } else { + check(); + + // Value does not exist, so we delete from current updates + currentUpdates.deletedKeys.add(keyW); + } + } else { + // pruning disabled + if (value != null) { + src.put(key, value); + } else { + check(); + } + } + } catch (Exception e) { + if (e instanceof RuntimeException) { + throw e; + } else { + LOG.error("Could not put key-value pair due to ", e); } - // delete is not sent to source db + } finally { + lock.writeLock().unlock(); } } - public synchronized void delete(byte[] key) { - if (!enabled) { + public void delete(byte[] key) { + checkNotNull(key); + if (!enabled.get()) { + check(); return; } - currentUpdates.deletedKeys.add(new ByteArrayWrapper(key)); - // delete is delayed + + lock.writeLock().lock(); + + try { + check(); + + currentUpdates.deletedKeys.add(ByteArrayWrapper.wrap(key)); + // delete is delayed + } catch (Exception e) { + if (e instanceof RuntimeException) { + throw e; + } else { + LOG.error("Could not delete key due to ", e); + } + } finally { + lock.writeLock().unlock(); + } } - public synchronized void updateBatch(Map rows) { - Map insertsOnly = new HashMap<>(); - for (Map.Entry entry : rows.entrySet()) { - ByteArrayWrapper keyW = new ByteArrayWrapper(entry.getKey()); - if (entry.getValue() != null) { - if (enabled) { - currentUpdates.insertedKeys.add(keyW); - incRef(keyW); + @Override + public void putBatch(Map inputMap) { + checkNotNull(inputMap.keySet()); + + lock.writeLock().lock(); + + try { + Map insertsOnly = new HashMap<>(); + if (enabled.get()) { + for (Map.Entry entry : inputMap.entrySet()) { + ByteArrayWrapper keyW = ByteArrayWrapper.wrap(entry.getKey()); + if (entry.getValue() != null) { + currentUpdates.insertedKeys.add(keyW); + incRef(keyW); + insertsOnly.put(entry.getKey(), entry.getValue()); + } else { + currentUpdates.deletedKeys.add(keyW); + } } - insertsOnly.put(entry.getKey(), entry.getValue()); } else { - if (enabled) { - currentUpdates.deletedKeys.add(keyW); + for (Map.Entry entry : inputMap.entrySet()) { + if (entry.getValue() != null) { + insertsOnly.put(entry.getKey(), entry.getValue()); + } } } + src.putBatch(insertsOnly); + } catch (Exception e) { + if (e instanceof RuntimeException) { + throw e; + } else { + LOG.error("Could not put batch due to ", e); + } + } finally { + lock.writeLock().unlock(); } - src.putBatch(insertsOnly); } private void incRef(ByteArrayWrapper keyW) { Ref cnt = refCount.get(keyW); if (cnt == null) { - cnt = new Ref(src.get(keyW.getData()) != null); + cnt = new Ref(src.get(keyW.getData()).isPresent()); refCount.put(keyW, cnt); } cnt.journalRefs++; @@ -157,60 +231,75 @@ private Ref decRef(ByteArrayWrapper keyW) { return cnt; } - public synchronized void storeBlockChanges(BH header) { - if (!enabled) { + public void storeBlockChanges(byte[] blockHash, long blockNumber) { + if (!enabled.get()) { return; } - currentUpdates.blockHeader = header; - blockUpdates.put(new ByteArrayWrapper(header.getHash()), currentUpdates); - currentUpdates = new Updates(); + + lock.writeLock().lock(); + + try { + ByteArrayWrapper hash = ByteArrayWrapper.wrap(blockHash); + currentUpdates.blockHeader = hash; + currentUpdates.blockNumber = blockNumber; + blockUpdates.put(hash, currentUpdates); + currentUpdates = new Updates(); + } finally { + lock.writeLock().unlock(); + } } - public synchronized void prune(BH header) { - if (!enabled) { + public void prune(byte[] blockHash, long blockNumber) { + if (!enabled.get()) { return; } - ByteArrayWrapper blockHashW = new ByteArrayWrapper(header.getHash()); - Updates updates = blockUpdates.remove(blockHashW); - if (updates != null) { - for (ByteArrayWrapper insertedKey : updates.insertedKeys) { - decRef(insertedKey).dbRef = true; - } - List batchRemove = new ArrayList<>(); - for (ByteArrayWrapper key : updates.deletedKeys) { - Ref ref = refCount.get(key); - if (ref == null || ref.journalRefs == 0) { - batchRemove.add(key.getData()); - } else if (ref != null) { - ref.dbRef = false; + lock.writeLock().lock(); + + try { + ByteArrayWrapper blockHashW = ByteArrayWrapper.wrap(blockHash); + Updates updates = blockUpdates.remove(blockHashW); + if (updates != null) { + for (ByteArrayWrapper insertedKey : updates.insertedKeys) { + decRef(insertedKey).dbRef = true; } - } - src.deleteBatch(batchRemove); - rollbackForkBlocks(header.getNumber()); + List batchRemove = new ArrayList<>(); + for (ByteArrayWrapper key : updates.deletedKeys) { + Ref ref = refCount.get(key); + if (ref == null || ref.journalRefs == 0) { + batchRemove.add(key.getData()); + } else if (ref != null) { + ref.dbRef = false; + } + } + src.deleteBatch(batchRemove); + + rollbackForkBlocks(blockNumber); + } + } finally { + lock.writeLock().unlock(); } } private void rollbackForkBlocks(long blockNum) { for (Updates updates : new ArrayList<>(blockUpdates.values())) { - if (updates.blockHeader.getNumber() == blockNum) { + if (updates.blockNumber == blockNum) { rollback(updates.blockHeader); } } } - private synchronized void rollback(BH header) { - ByteArrayWrapper blockHashW = new ByteArrayWrapper(header.getHash()); + private void rollback(ByteArrayWrapper blockHashW) { Updates updates = blockUpdates.remove(blockHashW); - Map batchRemove = new HashMap<>(); + List batchRemove = new ArrayList<>(); for (ByteArrayWrapper insertedKey : updates.insertedKeys) { Ref ref = decRef(insertedKey); if (ref.getTotRefs() == 0) { - batchRemove.put(insertedKey.getData(), null); + batchRemove.add(insertedKey.getData()); } } - src.putBatch(batchRemove); + src.deleteBatch(batchRemove); } public Map getRefCount() { @@ -222,29 +311,58 @@ public LinkedHashMap getBlockUpdates() { } public int getDeletedKeysCount() { - return currentUpdates.deletedKeys.size(); + lock.readLock().lock(); + try { + return currentUpdates.deletedKeys.size(); + } finally { + lock.readLock().unlock(); + } } public int getInsertedKeysCount() { - return currentUpdates.insertedKeys.size(); + lock.readLock().lock(); + try { + return currentUpdates.insertedKeys.size(); + } finally { + lock.readLock().unlock(); + } } public Optional get(byte[] key) { - return src.get(key); + lock.readLock().lock(); + try { + return src.get(key); + } catch (Exception e) { + LOG.error("Could not get key due to ", e); + throw e; + } finally { + lock.readLock().unlock(); + } } public Set keys() { - return src.keys(); + lock.readLock().lock(); + try { + return src.keys(); + } catch (Exception e) { + LOG.error("Could not get keys due to ", e); + throw e; + } finally { + lock.readLock().unlock(); + } } @Override public void close() { - src.close(); - } - - @Override - public void putBatch(Map inputMap) { - updateBatch(inputMap); + lock.writeLock().lock(); + + try { + src.close(); + } catch (Exception e) { + LOG.error("Could not close source due to ", e); + } finally { + lock.writeLock().unlock(); + } } @Override @@ -259,24 +377,84 @@ public void commitBatch() { @Override public void deleteBatch(Collection keys) { - if (!enabled) { + checkNotNull(keys); + if (!enabled.get()) { + check(); return; } - // deletes are delayed - keys.forEach(key -> currentUpdates.deletedKeys.add(new ByteArrayWrapper(key))); + + lock.writeLock().lock(); + + try { + check(); + + // deletes are delayed + keys.forEach(key -> currentUpdates.deletedKeys.add(ByteArrayWrapper.wrap(key))); + } catch (Exception e) { + if (e instanceof RuntimeException) { + throw e; + } else { + LOG.error("Could not delete batch due to ", e); + } + } finally { + lock.writeLock().unlock(); + } } @Override public boolean isEmpty() { - // the delayed deletes are not considered by this check until applied to the db - if (!currentUpdates.insertedKeys.isEmpty()) { - return false; - } else { - return src.isEmpty(); + lock.readLock().lock(); + + try { + // the delayed deletes are not considered by this check until applied to the db + if (!currentUpdates.insertedKeys.isEmpty()) { + check(); + return false; + } else { + return src.isEmpty(); + } + } catch (Exception e) { + LOG.error("Could not check if empty due to ", e); + throw e; + } finally { + lock.readLock().unlock(); } } - public IByteArrayKeyValueDatabase getSrc() { + public IByteArrayKeyValueStore getSrc() { return src; } + + public IByteArrayKeyValueDatabase getArchiveSource() { + if (!hasArchive) { + return null; + } else { + return ((ArchivedDataSource) src).getArchiveDatabase(); + } + } + + @Override + public void check() { + src.check(); + } + + /** + * Checks that the given key is not null. Throws a {@link IllegalArgumentException} if the key + * is null. + */ + public static void checkNotNull(byte[] k) { + if (k == null) { + throw new IllegalArgumentException("The data store does not accept null keys."); + } + } + + /** + * Checks that the given collection of keys does not contain null values. Throws a {@link + * IllegalArgumentException} if a null key is present. + */ + public static void checkNotNull(Collection keys) { + if (keys.contains(null)) { + throw new IllegalArgumentException("The data store does not accept null keys."); + } + } } diff --git a/modMcf/src/org/aion/mcf/trie/Trie.java b/modMcf/src/org/aion/mcf/trie/Trie.java index 8a0f865c30..34b38e312b 100644 --- a/modMcf/src/org/aion/mcf/trie/Trie.java +++ b/modMcf/src/org/aion/mcf/trie/Trie.java @@ -1,36 +1,45 @@ -/******************************************************************************* +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. * - * Copyright (c) 2017, 2018 Aion foundation. + * This file is part of the aion network project. * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see + * along with the aion network project source files. + * If not, see . * - * Contributors: + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: * Aion foundation. *******************************************************************************/ package org.aion.mcf.trie; +import org.aion.base.db.IByteArrayKeyValueDatabase; + /** - * Trie interface for the main data structure in Ethereum - * which is used to store both the account state and storage of each account. + * Trie interface for the main data structure in Ethereum which is used to store both the account + * state and storage of each account. */ public interface Trie { /** * Gets a value from the trie for a given key * - * @param key - * - any length byte array + * @param key - any length byte array * @return an rlp encoded byte array of the stored object */ byte[] get(byte[] key); @@ -38,18 +47,15 @@ public interface Trie { /** * Insert or update a value in the trie for a specified key * - * @param key - * - any length byte array - * @param value - * rlp encoded byte array of the object to store + * @param key - any length byte array + * @param value rlp encoded byte array of the object to store */ void update(byte[] key, byte[] value); /** * Deletes a key/value from the trie for a given key * - * @param key - * - any length byte array + * @param key - any length byte array */ void delete(byte[] key); @@ -63,37 +69,36 @@ public interface Trie { /** * Set the top node of the trie * - * @param root - * - 32-byte SHA-3 hash of the root node + * @param root - 32-byte SHA-3 hash of the root node */ void setRoot(byte[] root); /** * Used to check for corruption in the database. * - * @param root - * a world state trie root + * @param root a world state trie root * @return {@code true} if the root is valid, {@code false} otherwise */ boolean isValidRoot(byte[] root); - /** - * Commit all the changes until now - */ + /** Commit all the changes until now */ void sync(); void sync(boolean flushCache); - /** - * Discard all the changes until now - */ + /** Discard all the changes until now */ @Deprecated void undo(); String getTrieDump(); + String getTrieDump(byte[] stateRoot); + int getTrieSize(byte[] stateRoot); boolean validate(); -} \ No newline at end of file + long saveFullStateToDatabase(byte[] stateRoot, IByteArrayKeyValueDatabase db); + + long saveDiffStateToDatabase(byte[] stateRoot, IByteArrayKeyValueDatabase db); +} diff --git a/modMcf/src/org/aion/mcf/trie/TrieImpl.java b/modMcf/src/org/aion/mcf/trie/TrieImpl.java index 7f459824f5..4b3ef2ff2b 100644 --- a/modMcf/src/org/aion/mcf/trie/TrieImpl.java +++ b/modMcf/src/org/aion/mcf/trie/TrieImpl.java @@ -1,77 +1,78 @@ -/******************************************************************************* +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. * - * Copyright (c) 2017, 2018 Aion foundation. + * This file is part of the aion network project. * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see + * along with the aion network project source files. + * If not, see . * - * Contributors: + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: * Aion foundation. *******************************************************************************/ package org.aion.mcf.trie; +import static java.util.Arrays.copyOfRange; +import static org.aion.base.util.ByteArrayWrapper.wrap; +import static org.aion.base.util.ByteUtil.EMPTY_BYTE_ARRAY; +import static org.aion.base.util.ByteUtil.matchingNibbleLength; +import static org.aion.crypto.HashUtil.EMPTY_TRIE_HASH; +import static org.aion.rlp.CompactEncoder.*; +import static org.aion.rlp.RLP.calcElementPrefixSize; +import static org.spongycastle.util.Arrays.concatenate; + +import java.io.*; +import java.util.*; +import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.base.db.IByteArrayKeyValueStore; import org.aion.base.util.ByteArrayWrapper; import org.aion.base.util.FastByteComparisons; import org.aion.base.util.Hex; import org.aion.crypto.HashUtil; -import org.aion.mcf.trie.scan.CollectFullSetOfNodes; -import org.aion.mcf.trie.scan.CountNodes; -import org.aion.mcf.trie.scan.ScanAction; -import org.aion.mcf.trie.scan.TraceAllNodes; +import org.aion.mcf.trie.scan.*; import org.aion.rlp.RLP; import org.aion.rlp.RLPItem; import org.aion.rlp.RLPList; import org.aion.rlp.Value; -import java.io.*; -import java.util.*; - -import static java.util.Arrays.copyOfRange; -import static org.aion.base.util.ByteArrayWrapper.wrap; -import static org.aion.base.util.ByteUtil.EMPTY_BYTE_ARRAY; -import static org.aion.base.util.ByteUtil.matchingNibbleLength; -import static org.aion.crypto.HashUtil.EMPTY_TRIE_HASH; -import static org.aion.rlp.CompactEncoder.*; -import static org.aion.rlp.RLP.calcElementPrefixSize; -import static org.spongycastle.util.Arrays.concatenate; - /** - * The modified Merkle Patricia tree (trie) provides a persistent data structure - * to map between arbitrary-length binary data (byte arrays). It is defined in - * terms of a mutable data structure to map between 256-bit binary fragments and - * arbitrary-length binary data, typically implemented as a database. The core - * of the trie, and its sole requirement in terms of the protocol specification - * is to provide a single value that identifies a given set of key-value pairs, - * which may either a 32 byte sequence or the empty byte sequence. It is left as - * an implementation consideration to store and maintain the structure of the - * trie in a manner the allows effective and efficient realisation of the - * protocol. - *

- * The trie implements a caching mechanism and will use cached values if they - * are present. If a node is not present in the cache it will try to fetch it - * from the database and store the cached value. - *

- * Note: the data isn't persisted unless `sync` is explicitly called. - *

- * This Trie implementation supports node pruning (i.e. obsolete nodes are - * marked for removal in the Cache and actually removed from the underlying - * storage on [sync] call), but the algorithm is not suitable for the most - * general case. In general case a trie node might be referenced from several - * parent nodes and for correct pruning the reference counting algorithm needs - * to be implemented. As soon as the real life tree keys are hashes it is very - * unlikely the case so the pruning algorithm is simplified in this - * implementation. + * The modified Merkle Patricia tree (trie) provides a persistent data structure to map between + * arbitrary-length binary data (byte arrays). It is defined in terms of a mutable data structure to + * map between 256-bit binary fragments and arbitrary-length binary data, typically implemented as a + * database. The core of the trie, and its sole requirement in terms of the protocol specification + * is to provide a single value that identifies a given set of key-value pairs, which may either a + * 32 byte sequence or the empty byte sequence. It is left as an implementation consideration to + * store and maintain the structure of the trie in a manner the allows effective and efficient + * realisation of the protocol. + * + *

The trie implements a caching mechanism and will use cached values if they are present. If a + * node is not present in the cache it will try to fetch it from the database and store the cached + * value. + * + *

Note: the data isn't persisted unless `sync` is explicitly called. + * + *

This Trie implementation supports node pruning (i.e. obsolete nodes are marked for removal in + * the Cache and actually removed from the underlying storage on [sync] call), but the algorithm is + * not suitable for the most general case. In general case a trie node might be referenced from + * several parent nodes and for correct pruning the reference counting algorithm needs to be + * implemented. As soon as the real life tree keys are hashes it is very unlikely the case so the + * pruning algorithm is simplified in this implementation. * * @author Nick Savers * @since 20.05.2014 @@ -81,8 +82,7 @@ public class TrieImpl implements Trie { private static byte LIST_SIZE = 17; private static int MAX_SIZE = 20; - @Deprecated - private Object prevRoot; + @Deprecated private Object prevRoot; private Object root; private Cache cache; @@ -119,9 +119,7 @@ public Object getRoot() { return root; } - /** - * for testing TrieTest.testRollbackToRootScenarios - */ + /** for testing TrieTest.testRollbackToRootScenarios */ public void setRoot(Object root) { this.root = root; } @@ -152,9 +150,7 @@ public TrieImpl withPruningEnabled(boolean pruningEnabled) { return this; } - /** - * Retrieve a value from a key as String. - */ + /** Retrieve a value from a key as String. */ public byte[] get(String key) { return this.get(key.getBytes()); } @@ -169,9 +165,7 @@ public byte[] get(byte[] key) { } } - /** - * Insert key/value pair into trie. - */ + /** Insert key/value pair into trie. */ public void update(String key, String value) { this.update(key.getBytes(), value.getBytes()); } @@ -198,9 +192,7 @@ public synchronized boolean isValidRoot(byte[] root) { return !(this.getNode(root) == null); } - /** - * Delete a key/value pair from the trie. - */ + /** Delete a key/value pair from the trie. */ public void delete(String key) { this.update(key.getBytes(), EMPTY_BYTE_ARRAY); } @@ -215,8 +207,9 @@ public void delete(byte[] key) { @Override public byte[] getRootHash() { synchronized (cache) { - if (root == null || (root instanceof byte[] && ((byte[]) root).length == 0) || (root instanceof String && "" - .equals(root))) { + if (root == null + || (root instanceof byte[] && ((byte[]) root).length == 0) + || (root instanceof String && "".equals(root))) { return EMPTY_TRIE_HASH; } else if (root instanceof byte[]) { return (byte[]) this.getRoot(); @@ -241,8 +234,8 @@ private Object get(Object node, byte[] key) { byte[] k = unpackToNibbles(currentNode.get(0).asBytes()); Object v = currentNode.get(1).asObj(); - if (key.length - keypos >= k.length && Arrays - .equals(k, copyOfRange(key, keypos, k.length + keypos))) { + if (key.length - keypos >= k.length + && Arrays.equals(k, copyOfRange(key, keypos, k.length + keypos))) { node = v; keypos += k.length; } else { @@ -277,7 +270,7 @@ private Object insert(Object node, byte[] key, Object value) { } if (isEmptyNode(node)) { - Object[] newNode = new Object[] { packNibbles(key), value }; + Object[] newNode = new Object[] {packNibbles(key), value}; return this.putToCache(newNode); } @@ -295,7 +288,7 @@ private Object insert(Object node, byte[] key, Object value) { // Matching key pair (ie. there's already an object with this key) if (Arrays.equals(k, key)) { - Object[] newNode = new Object[] { packNibbles(key), value }; + Object[] newNode = new Object[] {packNibbles(key), value}; return this.putToCache(newNode); } @@ -311,7 +304,8 @@ private Object insert(Object node, byte[] key, Object value) { // Expand the 2 length slice to a 17 length slice // Create two nodes to putToCache into the new 17 length node Object oldNode = this.insert("", copyOfRange(k, matchingLength + 1, k.length), v); - Object newNode = this.insert("", copyOfRange(key, matchingLength + 1, key.length), value); + Object newNode = + this.insert("", copyOfRange(key, matchingLength + 1, key.length), value); // Create an expanded slice Object[] scaledSlice = emptyStringSlice(17); @@ -328,7 +322,8 @@ private Object insert(Object node, byte[] key, Object value) { // End of the chain, return return newHash; } else { - Object[] newNode = new Object[] { packNibbles(copyOfRange(key, 0, matchingLength)), newHash }; + Object[] newNode = + new Object[] {packNibbles(copyOfRange(key, 0, matchingLength)), newHash}; return this.putToCache(newNode); } } else { @@ -337,10 +332,15 @@ private Object insert(Object node, byte[] key, Object value) { Object[] newNode = copyNode(currentNode); // Replace the first nibble in the key - newNode[key[0]] = this.insert(currentNode.get(key[0]).asObj(), copyOfRange(key, 1, key.length), value); - - if (!FastByteComparisons - .equal(HashUtil.h256(getNode(newNode).encode()), HashUtil.h256(currentNode.encode()))) { + newNode[key[0]] = + this.insert( + currentNode.get(key[0]).asObj(), + copyOfRange(key, 1, key.length), + value); + + if (!FastByteComparisons.equal( + HashUtil.h256(getNode(newNode).encode()), + HashUtil.h256(currentNode.encode()))) { markRemoved(HashUtil.h256(currentNode.encode())); if (!isEmptyNode(currentNode.get(key[0]))) { markRemoved(currentNode.get(key[0]).asBytes()); @@ -379,9 +379,9 @@ private Object delete(Object node, byte[] key) { Object newNode; if (child.length() == PAIR_SIZE) { byte[] newKey = concatenate(k, unpackToNibbles(child.get(0).asBytes())); - newNode = new Object[] { packNibbles(newKey), child.get(1).asObj() }; + newNode = new Object[] {packNibbles(newKey), child.get(1).asObj()}; } else { - newNode = new Object[] { currentNode.get(0), hash }; + newNode = new Object[] {currentNode.get(0), hash}; } markRemoved(HashUtil.h256(currentNode.encode())); return this.putToCache(newNode); @@ -408,21 +408,22 @@ private Object delete(Object node, byte[] key) { Object[] newNode = null; if (amount == 16) { - newNode = new Object[] { packNibbles(new byte[] { 16 }), itemList[amount] }; + newNode = new Object[] {packNibbles(new byte[] {16}), itemList[amount]}; } else if (amount >= 0) { Value child = this.getNode(itemList[amount]); if (child.length() == PAIR_SIZE) { - key = concatenate(new byte[] { amount }, unpackToNibbles(child.get(0).asBytes())); - newNode = new Object[] { packNibbles(key), child.get(1).asObj() }; + key = concatenate(new byte[] {amount}, unpackToNibbles(child.get(0).asBytes())); + newNode = new Object[] {packNibbles(key), child.get(1).asObj()}; } else if (child.length() == LIST_SIZE) { - newNode = new Object[] { packNibbles(new byte[] { amount }), itemList[amount] }; + newNode = new Object[] {packNibbles(new byte[] {amount}), itemList[amount]}; } } else { newNode = itemList; } - if (!FastByteComparisons - .equal(HashUtil.h256(getNode(newNode).encode()), HashUtil.h256(currentNode.encode()))) { + if (!FastByteComparisons.equal( + HashUtil.h256(getNode(newNode).encode()), + HashUtil.h256(currentNode.encode()))) { markRemoved(HashUtil.h256(currentNode.encode())); } @@ -437,8 +438,8 @@ private void markRemoved(byte[] hash) { } /** - * Helper method to retrieve the actual node. If the node is not a list and - * length is > 32 bytes get the actual node from the db. + * Helper method to retrieve the actual node. If the node is not a list and length is > 32 bytes + * get the actual node from the db. */ private Value getNode(Object node) { @@ -465,7 +466,9 @@ private Object putToCache(Object node) { private boolean isEmptyNode(Object node) { Value n = new Value(node); - return (node == null || (n.isString() && (n.asString().isEmpty() || n.get(0).isNull())) || n.length() == 0); + return (node == null + || (n.isString() && (n.asString().isEmpty() || n.get(0).isNull())) + || n.length() == 0); } private Object[] copyNode(Value currentNode) { @@ -485,7 +488,8 @@ public boolean equals(Object trie) { if (this == trie) { return true; } - return trie instanceof Trie && Arrays.equals(this.getRootHash(), ((Trie) trie).getRootHash()); + return trie instanceof Trie + && Arrays.equals(this.getRootHash(), ((Trie) trie).getRootHash()); } @Override @@ -524,10 +528,7 @@ public TrieImpl copy() { } } - /** - * ****************************** Utility functions * - * ***************************** - */ + /** ****************************** Utility functions * ***************************** */ // Created an array of empty elements of required length private static Object[] emptyStringSlice(int l) { Object[] slice = new Object[l]; @@ -538,13 +539,12 @@ private static Object[] emptyStringSlice(int l) { } /** - * Insert/delete operations on a Trie structure leaves the old nodes in - * cache, this method scans the cache and removes them. The method is not - * thread safe, the tree should not be modified during the cleaning process. + * Insert/delete operations on a Trie structure leaves the old nodes in cache, this method scans + * the cache and removes them. The method is not thread safe, the tree should not be modified + * during the cleaning process. */ public void cleanCache() { synchronized (cache) { - CollectFullSetOfNodes collectAction = new CollectFullSetOfNodes(); this.scanTree(this.getRootHash(), collectAction); @@ -576,7 +576,6 @@ public void printFootPrint() { public void scanTree(byte[] hash, ScanAction scanAction) { synchronized (cache) { - Value node = this.getCache().get(hash); if (node == null) { throw new RuntimeException("Not found: " + Hex.toHexString(hash)); @@ -638,6 +637,57 @@ public void scanTreeLoop(byte[] hash, ScanAction scanAction) { } } + /** + * Scans the trie similar to {@link #scanTreeLoop(byte[], ScanAction)}, but stops once a state + * is found in the given database. + * + * @param hash state root + * @param scanAction action to perform on each node + * @param db database containing keys that need not be explored + */ + public void scanTreeDiffLoop( + byte[] hash, ScanAction scanAction, IByteArrayKeyValueDatabase db) { + + ArrayList hashes = new ArrayList<>(); + hashes.add(hash); + + while (!hashes.isEmpty()) { + synchronized (cache) { + byte[] myHash = hashes.remove(0); + Value node = this.getCache().get(myHash); + if (node == null) { + System.out.println("Skipped key. Not found: " + Hex.toHexString(myHash)); + } else { + if (node.isList()) { + List siblings = node.asList(); + if (siblings.size() == PAIR_SIZE) { + Value val = new Value(siblings.get(1)); + if (val.isHashCode() && !hasTerminator((byte[]) siblings.get(0))) { + // scanTree(val.asBytes(), scanAction); + byte[] valBytes = val.asBytes(); + if (!db.get(valBytes).isPresent()) { + hashes.add(valBytes); + } + } + } else { + for (int j = 0; j < LIST_SIZE; ++j) { + Value val = new Value(siblings.get(j)); + if (val.isHashCode()) { + // scanTree(val.asBytes(), scanAction); + byte[] valBytes = val.asBytes(); + if (!db.get(valBytes).isPresent()) { + hashes.add(valBytes); + } + } + } + } + scanAction.doOnNode(myHash, node); + } + } + } + } + } + public void deserialize(byte[] data) { synchronized (cache) { RLPList rlpList = (RLPList) RLP.decode2(data).get(0); @@ -695,21 +745,43 @@ public byte[] serialize() { byte[] keysHeader = RLP.encodeLongElementHeader(keysTotalSize); byte[] valsHeader = RLP.encodeListHeader(valsTotalSize); - byte[] listHeader = RLP.encodeListHeader( - keysTotalSize + keysHeader.length + valsTotalSize + valsHeader.length + root.length); - - byte[] rlpData = new byte[keysTotalSize + keysHeader.length + valsTotalSize + valsHeader.length - + listHeader.length + root.length]; + byte[] listHeader = + RLP.encodeListHeader( + keysTotalSize + + keysHeader.length + + valsTotalSize + + valsHeader.length + + root.length); + + byte[] rlpData = + new byte + [keysTotalSize + + keysHeader.length + + valsTotalSize + + valsHeader.length + + listHeader.length + + root.length]; // copy headers: // [ rlp_list_header, rlp_keys_header, rlp_keys, rlp_vals_header, // rlp_val] System.arraycopy(listHeader, 0, rlpData, 0, listHeader.length); System.arraycopy(keysHeader, 0, rlpData, listHeader.length, keysHeader.length); - System.arraycopy(valsHeader, 0, rlpData, (listHeader.length + keysHeader.length + keysTotalSize), + System.arraycopy( + valsHeader, + 0, + rlpData, + (listHeader.length + keysHeader.length + keysTotalSize), valsHeader.length); - System.arraycopy(root, 0, rlpData, - (listHeader.length + keysHeader.length + keysTotalSize + valsTotalSize + valsHeader.length), + System.arraycopy( + root, + 0, + rlpData, + (listHeader.length + + keysHeader.length + + keysTotalSize + + valsTotalSize + + valsHeader.length), root.length); int k_1 = 0; @@ -720,15 +792,26 @@ public byte[] serialize() { continue; } - System.arraycopy(key.getData(), 0, rlpData, (listHeader.length + keysHeader.length + k_1), + System.arraycopy( + key.getData(), + 0, + rlpData, + (listHeader.length + keysHeader.length + k_1), key.getData().length); k_1 += key.getData().length; byte[] valBytes = RLP.encodeElement(node.getValue().getData()); - System.arraycopy(valBytes, 0, rlpData, - listHeader.length + keysHeader.length + keysTotalSize + valsHeader.length + k_2, + System.arraycopy( + valBytes, + 0, + rlpData, + listHeader.length + + keysHeader.length + + keysTotalSize + + valsHeader.length + + k_2, valBytes.length); k_2 += valBytes.length; } @@ -793,12 +876,14 @@ public boolean validate() { synchronized (cache) { final int[] cnt = new int[1]; try { - scanTree(getRootHash(), new ScanAction() { - @Override - public void doOnNode(byte[] hash, Value node) { - cnt[0]++; - } - }); + scanTree( + getRootHash(), + new ScanAction() { + @Override + public void doOnNode(byte[] hash, Value node) { + cnt[0]++; + } + }); } catch (Exception e) { return false; } @@ -806,4 +891,29 @@ public void doOnNode(byte[] hash, Value node) { } } + @Override + public long saveFullStateToDatabase(byte[] stateRoot, IByteArrayKeyValueDatabase db) { + ExtractToDatabase traceAction = new ExtractToDatabase(db); + traceTrie(stateRoot, traceAction); + return traceAction.count; + } + + private void traceDiffTrie(byte[] stateRoot, ScanAction action, IByteArrayKeyValueDatabase db) { + synchronized (cache) { + Value value = new Value(stateRoot); + + if (value.isHashCode() && !db.get(value.asBytes()).isPresent()) { + scanTreeDiffLoop(stateRoot, action, db); + } else { + action.doOnNode(stateRoot, value); + } + } + } + + @Override + public long saveDiffStateToDatabase(byte[] stateRoot, IByteArrayKeyValueDatabase db) { + ExtractToDatabase traceAction = new ExtractToDatabase(db); + traceDiffTrie(stateRoot, traceAction, db); + return traceAction.count; + } } diff --git a/modMcf/src/org/aion/mcf/trie/scan/ExtractToDatabase.java b/modMcf/src/org/aion/mcf/trie/scan/ExtractToDatabase.java new file mode 100644 index 0000000000..bb617f82f5 --- /dev/null +++ b/modMcf/src/org/aion/mcf/trie/scan/ExtractToDatabase.java @@ -0,0 +1,51 @@ +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: + * Aion foundation. + ******************************************************************************/ +package org.aion.mcf.trie.scan; + +import org.aion.base.db.IByteArrayKeyValueDatabase; +import org.aion.rlp.Value; + +/** @author Alexandra Roatis */ +public class ExtractToDatabase implements ScanAction { + + // only the keys are relevant so the value will be this constant + byte[] dummy_value = new byte[] {0}; + IByteArrayKeyValueDatabase db; + public long count = 0; + + public ExtractToDatabase(IByteArrayKeyValueDatabase _db) { + this.db = _db; + } + + @Override + public void doOnNode(byte[] hash, Value node) { + db.put(hash, dummy_value); + count++; + } +} diff --git a/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java b/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java new file mode 100644 index 0000000000..a32657474b --- /dev/null +++ b/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java @@ -0,0 +1,1561 @@ +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: + * Aion foundation. + ******************************************************************************/ +package org.aion.mcf.trie; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertTrue; + +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.aion.base.db.IByteArrayKeyValueDatabase; +import org.aion.db.impl.DatabaseFactory; +import org.aion.log.AionLoggerFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** @author Alexandra Roatis */ +public class JournalPruneDataSourceTest { + + private static final String dbName = "TestDB"; + private static IByteArrayKeyValueDatabase source_db = DatabaseFactory.connect(dbName); + private static JournalPruneDataSource db; + + private static final byte[] k1 = "key1".getBytes(); + private static final byte[] v1 = "value1".getBytes(); + + private static final byte[] k2 = "key2".getBytes(); + private static final byte[] v2 = "value2".getBytes(); + + private static final byte[] k3 = "key3".getBytes(); + private static final byte[] v3 = "value3".getBytes(); + + @BeforeClass + public static void setup() { + // logging to see errors + Map cfg = new HashMap<>(); + cfg.put("DB", "INFO"); + + AionLoggerFactory.init(cfg); + } + + @Before + public void open() { + assertThat(source_db.open()).isTrue(); + db = new JournalPruneDataSource(source_db); + } + + @After + public void close() { + source_db.close(); + assertThat(source_db.isClosed()).isTrue(); + } + + // Pruning disabled tests ---------------------------------------------------- + + @Test + public void testPut_woPrune() { + db.setPruneEnabled(false); + + assertThat(db.get(k1).isPresent()).isFalse(); + db.put(k1, v1); + assertThat(db.get(k1).get()).isEqualTo(v1); + + // ensure no cached values + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + + // ensure the insert was propagated + assertThat(source_db.get(k1).get()).isEqualTo(v1); + } + + @Test + public void testPutBatch_woPrune() { + db.setPruneEnabled(false); + + assertThat(db.get(k1).isPresent()).isFalse(); + assertThat(db.get(k2).isPresent()).isFalse(); + + Map map = new HashMap<>(); + map.put(k1, v1); + map.put(k2, v2); + db.putBatch(map); + + assertThat(v1).isEqualTo(db.get(k1).get()); + assertThat(v2).isEqualTo(db.get(k2).get()); + + // ensure no cached values + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + + // ensure the inserts were propagated + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + } + + @Test + public void testUpdate_woPrune() { + db.setPruneEnabled(false); + + // insert + assertThat(db.get(k1).isPresent()).isFalse(); + db.put(k1, v1); + assertThat(db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k1).get()).isEqualTo(v1); + + // update + db.put(k1, v2); + assertThat(db.get(k1).get()).isEqualTo(v2); + assertThat(source_db.get(k1).get()).isEqualTo(v2); + + // indirect delete + db.put(k1, null); + assertThat(db.get(k1).isPresent()).isTrue(); + assertThat(source_db.get(k1).isPresent()).isTrue(); + + // direct delete + db.put(k2, v2); + assertThat(db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + db.delete(k2); + assertThat(db.get(k2).isPresent()).isTrue(); + assertThat(source_db.get(k2).isPresent()).isTrue(); + + // ensure no cached values + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + } + + @Test + public void testUpdateBatch_woPrune() { + db.setPruneEnabled(false); + + // ensure existence + assertThat(db.get(k1).isPresent()).isFalse(); + assertThat(db.get(k2).isPresent()).isFalse(); + assertThat(db.get(k3).isPresent()).isFalse(); + db.put(k1, v1); + db.put(k2, v2); + + assertThat(v1).isEqualTo(db.get(k1).get()); + assertThat(v2).isEqualTo(db.get(k2).get()); + + // check after update + Map ops = new HashMap<>(); + ops.put(k1, null); + ops.put(k2, v1); + ops.put(k3, v3); + db.putBatch(ops); + + assertThat(db.get(k1).isPresent()).isTrue(); + assertThat(db.get(k2).get()).isEqualTo(v1); + assertThat(db.get(k3).get()).isEqualTo(v3); + + assertThat(source_db.get(k1).isPresent()).isTrue(); + assertThat(source_db.get(k2).get()).isEqualTo(v1); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // ensure no cached values + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + } + + @Test + public void testDelete_woPrune() { + db.setPruneEnabled(false); + + // ensure existence + db.put(k1, v1); + assertThat(db.get(k1).isPresent()).isTrue(); + + // delete not propagated + db.delete(k1); + assertThat(db.get(k1).isPresent()).isTrue(); + assertThat(source_db.get(k1).get()).isEqualTo(v1); + + // ensure no cached values + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + @Test + public void testDeleteBatch_woPrune() { + db.setPruneEnabled(false); + + // ensure existence + Map map = new HashMap<>(); + map.put(k1, v1); + map.put(k2, v2); + map.put(k3, null); + db.putBatch(map); + + assertThat(db.get(k1).isPresent()).isTrue(); + assertThat(db.get(k2).isPresent()).isTrue(); + assertThat(db.get(k3).isPresent()).isFalse(); + + // deletes not propagated + db.deleteBatch(map.keySet()); + assertThat(db.get(k1).isPresent()).isTrue(); + assertThat(db.get(k2).isPresent()).isTrue(); + assertThat(db.get(k3).isPresent()).isFalse(); + + // ensure no cached values + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + @Test + public void testKeys_woPrune() { + db.setPruneEnabled(false); + + // keys shouldn't be null even when empty + Set keys = db.keys(); + assertThat(db.isEmpty()).isTrue(); + assertThat(keys).isNotNull(); + assertThat(keys.size()).isEqualTo(0); + + // checking after put + db.put(k1, v1); + db.put(k2, v2); + assertThat(db.get(k1).get()).isEqualTo(v1); + assertThat(db.get(k2).get()).isEqualTo(v2); + + keys = db.keys(); + assertThat(keys.size()).isEqualTo(2); + + // checking after delete + db.delete(k2); + assertThat(db.get(k2).isPresent()).isTrue(); + + keys = db.keys(); + assertThat(keys.size()).isEqualTo(2); + + // checking after putBatch + Map ops = new HashMap<>(); + ops.put(k1, null); + ops.put(k2, v2); + ops.put(k3, v3); + db.putBatch(ops); + + keys = db.keys(); + assertThat(keys.size()).isEqualTo(3); + + // checking after deleteBatch + db.deleteBatch(ops.keySet()); + + keys = db.keys(); + assertThat(keys.size()).isEqualTo(3); + + // ensure no cached values + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + @Test + public void testIsEmpty_woPrune() { + db.setPruneEnabled(false); + + assertThat(db.isEmpty()).isTrue(); + + // checking after put + db.put(k1, v1); + db.put(k2, v2); + assertThat(db.get(k1).get()).isEqualTo(v1); + assertThat(db.get(k2).get()).isEqualTo(v2); + + assertThat(db.isEmpty()).isFalse(); + + // checking after delete + db.delete(k2); + assertThat(db.get(k2).isPresent()).isTrue(); + assertThat(db.isEmpty()).isFalse(); + db.delete(k1); + assertThat(db.isEmpty()).isFalse(); + + // checking after putBatch + Map ops = new HashMap<>(); + ops.put(k1, null); + ops.put(k2, v2); + ops.put(k3, v3); + db.putBatch(ops); + assertThat(db.isEmpty()).isFalse(); + + // checking after deleteBatch + db.deleteBatch(ops.keySet()); + assertThat(db.isEmpty()).isFalse(); + + // ensure no cached values + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + // Pruning enabled tests ---------------------------------------------------- + + private static final byte[] b0 = "block0".getBytes(); + + @Test + public void testPut_wPrune() { + db.setPruneEnabled(true); + + assertThat(db.get(k1).isPresent()).isFalse(); + db.put(k1, v1); + assertThat(db.get(k1).get()).isEqualTo(v1); + + // ensure cached values + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + + // ensure the insert was propagated + assertThat(source_db.get(k1).get()).isEqualTo(v1); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + @Test + public void testPutBatch_wPrune() { + db.setPruneEnabled(true); + + assertThat(db.get(k1).isPresent()).isFalse(); + assertThat(db.get(k2).isPresent()).isFalse(); + + Map map = new HashMap<>(); + map.put(k1, v1); + map.put(k2, v2); + db.putBatch(map); + + assertThat(v1).isEqualTo(db.get(k1).get()); + assertThat(v2).isEqualTo(db.get(k2).get()); + + // ensure cached values + assertThat(db.getInsertedKeysCount()).isEqualTo(2); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + + // ensure the inserts were propagated + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + @Test + public void testUpdate_wPrune() { + db.setPruneEnabled(true); + + // insert + assertThat(db.get(k1).isPresent()).isFalse(); + db.put(k1, v1); + assertThat(db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k1).get()).isEqualTo(v1); + + // update + db.put(k1, v2); + assertThat(db.get(k1).get()).isEqualTo(v2); + assertThat(source_db.get(k1).get()).isEqualTo(v2); + + // indirect delete + db.put(k1, null); + assertThat(db.get(k1).isPresent()).isTrue(); + assertThat(source_db.get(k1).isPresent()).isTrue(); + + // direct delete + db.put(k2, v2); + assertThat(db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + db.delete(k2); + assertThat(db.get(k2).isPresent()).isTrue(); + assertThat(source_db.get(k2).isPresent()).isTrue(); + + // ensure cached values + assertThat(db.getDeletedKeysCount()).isEqualTo(2); + assertThat(db.getInsertedKeysCount()).isEqualTo(2); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + @Test + public void testUpdateBatch_wPrune() { + db.setPruneEnabled(true); + + // ensure existence + assertThat(db.get(k1).isPresent()).isFalse(); + assertThat(db.get(k2).isPresent()).isFalse(); + assertThat(db.get(k3).isPresent()).isFalse(); + db.put(k1, v1); + db.put(k2, v2); + + assertThat(v1).isEqualTo(db.get(k1).get()); + assertThat(v2).isEqualTo(db.get(k2).get()); + + // check after update + Map ops = new HashMap<>(); + ops.put(k1, null); + ops.put(k2, v1); + ops.put(k3, v3); + db.putBatch(ops); + + assertThat(db.get(k1).isPresent()).isTrue(); + assertThat(db.get(k2).get()).isEqualTo(v1); + assertThat(db.get(k3).get()).isEqualTo(v3); + + assertThat(source_db.get(k1).isPresent()).isTrue(); + assertThat(source_db.get(k2).get()).isEqualTo(v1); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // ensure cached values + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + assertThat(db.getInsertedKeysCount()).isEqualTo(3); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + @Test + public void testDelete_wPrune() { + db.setPruneEnabled(true); + + // ensure existence + db.put(k1, v1); + assertThat(db.get(k1).isPresent()).isTrue(); + + // delete not propagated + db.delete(k1); + assertThat(db.get(k1).isPresent()).isTrue(); + assertThat(source_db.get(k1).get()).isEqualTo(v1); + + // ensure cached values + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + @Test + public void testDeleteBatch_wPrune() { + db.setPruneEnabled(true); + + // ensure existence + Map map = new HashMap<>(); + map.put(k1, v1); + map.put(k2, v2); + map.put(k3, null); + db.putBatch(map); + + assertThat(db.get(k1).isPresent()).isTrue(); + assertThat(db.get(k2).isPresent()).isTrue(); + assertThat(db.get(k3).isPresent()).isFalse(); + + // check presence after delete + db.deleteBatch(map.keySet()); + + // delete operations are delayed till pruning is called + assertThat(db.get(k1).isPresent()).isTrue(); + assertThat(db.get(k2).isPresent()).isTrue(); + assertThat(db.get(k3).isPresent()).isFalse(); + + // ensure cached values + assertThat(db.getDeletedKeysCount()).isEqualTo(3); + assertThat(db.getInsertedKeysCount()).isEqualTo(2); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + @Test + public void testKeys_wPrune() { + db.setPruneEnabled(true); + + // keys shouldn't be null even when empty + Set keys = db.keys(); + assertThat(db.isEmpty()).isTrue(); + assertThat(keys).isNotNull(); + assertThat(keys.size()).isEqualTo(0); + + // checking after put + db.put(k1, v1); + db.put(k2, v2); + assertThat(db.get(k1).get()).isEqualTo(v1); + assertThat(db.get(k2).get()).isEqualTo(v2); + + keys = db.keys(); + assertThat(keys.size()).isEqualTo(2); + + // checking after delete + db.delete(k2); + assertThat(db.get(k2).isPresent()).isTrue(); + + keys = db.keys(); + assertThat(keys.size()).isEqualTo(2); + + // checking after putBatch + Map ops = new HashMap<>(); + ops.put(k1, null); + ops.put(k2, v2); + ops.put(k3, v3); + db.putBatch(ops); + + keys = db.keys(); + assertThat(keys.size()).isEqualTo(3); + + // checking after deleteBatch + db.deleteBatch(ops.keySet()); + + keys = db.keys(); + assertThat(keys.size()).isEqualTo(3); + + // ensure no cached values + assertThat(db.getInsertedKeysCount()).isEqualTo(3); + assertThat(db.getDeletedKeysCount()).isEqualTo(3); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + @Test + public void testIsEmpty_wPrune() { + db.setPruneEnabled(true); + + assertThat(db.isEmpty()).isTrue(); + + // checking after put + db.put(k1, v1); + db.put(k2, v2); + assertThat(db.get(k1).get()).isEqualTo(v1); + assertThat(db.get(k2).get()).isEqualTo(v2); + + assertThat(db.isEmpty()).isFalse(); + + // checking after delete + db.delete(k2); + assertThat(db.get(k2).isPresent()).isTrue(); + assertThat(db.isEmpty()).isFalse(); + db.delete(k1); + assertThat(db.isEmpty()).isFalse(); + + // checking after putBatch + Map ops = new HashMap<>(); + ops.put(k1, null); + ops.put(k2, v2); + ops.put(k3, v3); + db.putBatch(ops); + assertThat(db.isEmpty()).isFalse(); + + // checking after deleteBatch + db.deleteBatch(ops.keySet()); + assertThat(db.isEmpty()).isFalse(); + + // ensure no cached values + assertThat(db.getInsertedKeysCount()).isEqualTo(3); + assertThat(db.getDeletedKeysCount()).isEqualTo(3); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + // Access with exception tests ---------------------------------------------------- + + @Test(expected = RuntimeException.class) + public void testIsEmpty_wClosedDatabase() { + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + // attempt isEmpty on closed db + db.isEmpty(); + } + + @Test(expected = RuntimeException.class) + public void testIsEmpty_wClosedDatabase_wInsertedKeys() { + db.setPruneEnabled(true); + db.put(randomBytes(32), randomBytes(32)); + + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + + // attempt isEmpty on closed db + db.isEmpty(); + } + + @Test(expected = RuntimeException.class) + public void testKeys_wClosedDatabase() { + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + // attempt keys on closed db + db.keys(); + } + + @Test(expected = RuntimeException.class) + public void testGet_wClosedDatabase() { + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + // attempt get on closed db + db.get(randomBytes(32)); + } + + @Test(expected = RuntimeException.class) + public void testPut_wClosedDatabase() { + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + // attempt put on closed db + db.put(randomBytes(32), randomBytes(32)); + } + + @Test(expected = RuntimeException.class) + public void testPut_wClosedDatabase_wNullValue() { + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + // attempt put on closed db + db.put(randomBytes(32), null); + } + + @Test(expected = RuntimeException.class) + public void testDelete_wClosedDatabase_wPrune() { + db.setPruneEnabled(true); + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + // attempt delete on closed db + db.delete(randomBytes(32)); + } + + @Test(expected = RuntimeException.class) + public void testDelete_wClosedDatabase_woPrune() { + db.setPruneEnabled(false); + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + // attempt delete on closed db + db.delete(randomBytes(32)); + } + + @Test(expected = RuntimeException.class) + public void testPutBatch_wClosedDatabase() { + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + Map map = new HashMap<>(); + map.put(randomBytes(32), randomBytes(32)); + map.put(randomBytes(32), randomBytes(32)); + map.put(randomBytes(32), randomBytes(32)); + + // attempt putBatch on closed db + db.putBatch(map); + } + + @Test(expected = RuntimeException.class) + public void testDeleteBatch_wClosedDatabase_wPrune() { + db.setPruneEnabled(true); + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + List list = new ArrayList<>(); + list.add(randomBytes(32)); + list.add(randomBytes(32)); + list.add(randomBytes(32)); + + // attempt deleteBatch on closed db + db.deleteBatch(list); + } + + @Test(expected = RuntimeException.class) + public void testDeleteBatch_wClosedDatabase_woPrune() { + db.setPruneEnabled(false); + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + List list = new ArrayList<>(); + list.add(randomBytes(32)); + list.add(randomBytes(32)); + list.add(randomBytes(32)); + + // attempt deleteBatch on closed db + db.deleteBatch(list); + } + + @Test(expected = IllegalArgumentException.class) + public void testGet_wNullKey() { + assertThat(source_db.open()).isTrue(); + + // attempt get with null key + db.get(null); + } + + @Test(expected = IllegalArgumentException.class) + public void testPut_wNullKey() { + assertThat(source_db.open()).isTrue(); + + // attempt put with null key + db.put(null, randomBytes(32)); + } + + @Test(expected = IllegalArgumentException.class) + public void testDelete_wNullKey() { + assertThat(source_db.open()).isTrue(); + + // attempt delete with null key + db.delete(null); + } + + @Test(expected = IllegalArgumentException.class) + public void testPutBatch_wNullKey() { + assertThat(source_db.open()).isTrue(); + + Map map = new HashMap<>(); + map.put(randomBytes(32), randomBytes(32)); + map.put(randomBytes(32), randomBytes(32)); + map.put(null, randomBytes(32)); + + // attempt putBatch on closed db + db.putBatch(map); + } + + @Test(expected = IllegalArgumentException.class) + public void testDeleteBatch_wNullKey() { + assertThat(source_db.open()).isTrue(); + + List list = new ArrayList<>(); + list.add(randomBytes(32)); + list.add(randomBytes(32)); + list.add(null); + + // attempt deleteBatch on closed db + db.deleteBatch(list); + } + + public static byte[] randomBytes(int length) { + byte[] result = new byte[length]; + new Random().nextBytes(result); + return result; + } + + // Concurrent access tests ---------------------------------------------------- + + private static final int CONCURRENT_THREADS = 200; + private static final int TIME_OUT = 100; // in seconds + private static final boolean DISPLAY_MESSAGES = false; + + private void addThread_IsEmpty(List threads, JournalPruneDataSource db) { + threads.add( + () -> { + boolean check = db.isEmpty(); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + + ": " + + (check ? "EMPTY" : "NOT EMPTY")); + } + }); + } + + private void addThread_Keys(List threads, JournalPruneDataSource db) { + threads.add( + () -> { + Set keys = db.keys(); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + ": #keys = " + keys.size()); + } + }); + } + + private void addThread_Get(List threads, JournalPruneDataSource db, String key) { + threads.add( + () -> { + boolean hasValue = db.get(key.getBytes()).isPresent(); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + + ": " + + key + + " " + + (hasValue ? "PRESENT" : "NOT PRESENT")); + } + }); + } + + private void addThread_Put(List threads, JournalPruneDataSource db, String key) { + threads.add( + () -> { + db.put(key.getBytes(), randomBytes(32)); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + ": " + key + " ADDED"); + } + }); + } + + private void addThread_Delete(List threads, JournalPruneDataSource db, String key) { + threads.add( + () -> { + db.delete(key.getBytes()); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + ": " + key + " DELETED"); + } + }); + } + + private void addThread_PutBatch(List threads, JournalPruneDataSource db, String key) { + threads.add( + () -> { + Map map = new HashMap<>(); + map.put((key + 1).getBytes(), randomBytes(32)); + map.put((key + 2).getBytes(), randomBytes(32)); + map.put((key + 3).getBytes(), randomBytes(32)); + db.putBatch(map); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + + ": " + + (key + 1) + + ", " + + (key + 2) + + ", " + + (key + 3) + + " ADDED"); + } + }); + } + + private void addThread_DeleteBatch( + List threads, JournalPruneDataSource db, String key) { + threads.add( + () -> { + List list = new ArrayList<>(); + list.add((key + 1).getBytes()); + list.add((key + 2).getBytes()); + list.add((key + 3).getBytes()); + db.deleteBatch(list); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + + ": " + + (key + 1) + + ", " + + (key + 2) + + ", " + + (key + 3) + + " DELETED"); + } + }); + } + + private void addThread_StoreBlockChanges( + List threads, JournalPruneDataSource db, String hash, long number) { + threads.add( + () -> { + db.storeBlockChanges(hash.getBytes(), number); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + + ": block (" + + hash + + ", " + + number + + ") STORED"); + } + }); + } + + private void addThread_Prune( + List threads, JournalPruneDataSource db, String hash, long number) { + threads.add( + () -> { + db.prune(hash.getBytes(), number); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + + ": block (" + + hash + + ", " + + number + + ") PRUNED"); + } + }); + } + + @Test + public void testConcurrentAccessOnOpenDatabase() throws InterruptedException { + assertThat(source_db.isOpen()).isTrue(); + db.setPruneEnabled(true); + + // create distinct threads with + List threads = new ArrayList<>(); + + int threadSetCount = CONCURRENT_THREADS / 8; + if (threadSetCount < 3) { + threadSetCount = 3; + } + + String keyStr, blockStr; + + for (int i = 0; i < threadSetCount; i++) { + // 1. thread that checks empty + addThread_IsEmpty(threads, db); + + // 2. thread that gets keys + addThread_Keys(threads, db); + + keyStr = "key-" + i + "."; + + // 3. thread that gets entry + addThread_Get(threads, db, keyStr); + + // 4. thread that puts entry + addThread_Put(threads, db, keyStr); + + // 5. thread that deletes entry + addThread_Delete(threads, db, keyStr); + + // 6. thread that puts entries + addThread_PutBatch(threads, db, keyStr); + + // 7. thread that deletes entry + addThread_DeleteBatch(threads, db, keyStr); + + blockStr = "block-" + i + "."; + + // 8. thread that stores block changes + addThread_StoreBlockChanges(threads, db, blockStr, i); + + // 9. thread that prunes + addThread_Prune(threads, db, blockStr, i); + } + + // run threads and check for exceptions + assertConcurrent("Testing concurrent access. ", threads, TIME_OUT); + + // ensuring close + db.close(); + assertThat(source_db.isClosed()).isTrue(); + } + + @Test + public void testConcurrentPut() throws InterruptedException { + assertThat(source_db.isOpen()).isTrue(); + db.setPruneEnabled(true); + + // create distinct threads with + List threads = new ArrayList<>(); + + for (int i = 0; i < CONCURRENT_THREADS; i++) { + addThread_Put(threads, db, "key-" + i); + } + + // run threads + assertConcurrent("Testing put(...) ", threads, TIME_OUT); + + // check that all values were added + assertThat(db.keys().size()).isEqualTo(CONCURRENT_THREADS); + + // ensuring close + db.close(); + assertThat(source_db.isClosed()).isTrue(); + } + + @Test + public void testConcurrentPutBatch() throws InterruptedException { + assertThat(source_db.isOpen()).isTrue(); + db.setPruneEnabled(true); + + // create distinct threads with + List threads = new ArrayList<>(); + + for (int i = 0; i < CONCURRENT_THREADS; i++) { + addThread_PutBatch(threads, db, "key-" + i); + } + + // run threads + assertConcurrent("Testing putBatch(...) ", threads, TIME_OUT); + + // check that all values were added + assertThat(db.keys().size()).isEqualTo(3 * CONCURRENT_THREADS); + + // ensuring close + db.close(); + assertThat(source_db.isClosed()).isTrue(); + } + + @Test + public void testConcurrentUpdate() throws InterruptedException { + assertThat(source_db.isOpen()).isTrue(); + db.setPruneEnabled(true); + + // create distinct threads with + List threads = new ArrayList<>(); + + int threadSetCount = CONCURRENT_THREADS / 4; + if (threadSetCount < 3) { + threadSetCount = 3; + } + + String keyStr, blockStr; + + for (int i = 0; i < threadSetCount; i++) { + keyStr = "key-" + i + "."; + + // 1. thread that puts entry + addThread_Put(threads, db, keyStr); + + // 2. thread that deletes entry + addThread_Delete(threads, db, keyStr); + + // 3. thread that puts entries + addThread_PutBatch(threads, db, keyStr); + + // 4. thread that deletes entry + addThread_DeleteBatch(threads, db, keyStr); + + blockStr = "block-" + i + "."; + + // 5. thread that stores block changes + addThread_StoreBlockChanges(threads, db, blockStr, i); + + // 6. thread that prunes + addThread_Prune(threads, db, blockStr, i); + } + + // run threads and check for exceptions + assertConcurrent("Testing concurrent updates. ", threads, TIME_OUT); + + // ensuring close + db.close(); + assertThat(source_db.isClosed()).isTrue(); + } + + /** + * From JUnit + * Wiki on multithreaded code and concurrency + */ + public static void assertConcurrent( + final String message, + final List runnables, + final int maxTimeoutSeconds) + throws InterruptedException { + final int numThreads = runnables.size(); + final List exceptions = Collections.synchronizedList(new ArrayList()); + final ExecutorService threadPool = Executors.newFixedThreadPool(numThreads); + try { + final CountDownLatch allExecutorThreadsReady = new CountDownLatch(numThreads); + final CountDownLatch afterInitBlocker = new CountDownLatch(1); + final CountDownLatch allDone = new CountDownLatch(numThreads); + for (final Runnable submittedTestRunnable : runnables) { + threadPool.submit( + () -> { + allExecutorThreadsReady.countDown(); + try { + afterInitBlocker.await(); + submittedTestRunnable.run(); + } catch (final Throwable e) { + exceptions.add(e); + } finally { + allDone.countDown(); + } + }); + } + // wait until all threads are ready + assertTrue( + "Timeout initializing threads! Perform long lasting initializations before passing runnables to assertConcurrent", + allExecutorThreadsReady.await(runnables.size() * 10, TimeUnit.MILLISECONDS)); + // start all test runners + afterInitBlocker.countDown(); + assertTrue( + message + " timeout! More than" + maxTimeoutSeconds + "seconds", + allDone.await(maxTimeoutSeconds, TimeUnit.SECONDS)); + } finally { + threadPool.shutdownNow(); + } + if (!exceptions.isEmpty()) { + for (Throwable e : exceptions) { + e.printStackTrace(); + } + } + assertTrue( + message + "failed with " + exceptions.size() + " exception(s):" + exceptions, + exceptions.isEmpty()); + } + + // Pruning tests ---------------------------------------------------- + + private static final byte[] b1 = "block1".getBytes(); + private static final byte[] b2 = "block2".getBytes(); + private static final byte[] b3 = "block3".getBytes(); + + private static final byte[] k4 = "key4".getBytes(); + private static final byte[] v4 = "value4".getBytes(); + + private static final byte[] k5 = "key5".getBytes(); + private static final byte[] v5 = "value5".getBytes(); + + private static final byte[] k6 = "key6".getBytes(); + private static final byte[] v6 = "value6".getBytes(); + + @Test + public void pruningTest() { + db.setPruneEnabled(true); + + // block 0 + db.put(k1, v1); + db.put(k2, v2); + db.put(k3, v3); + assertThat(db.getInsertedKeysCount()).isEqualTo(3); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + db.storeBlockChanges(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // block 1 + db.put(k4, v4); + db.delete(k2); + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.get(k2).isPresent()).isTrue(); + assertThat(source_db.get(k4).get()).isEqualTo(v4); + + // block 2 + db.put(k2, v3); + db.delete(k3); + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b2, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).isPresent()).isTrue(); + + // block 3 + db.put(k5, v5); + db.put(k6, v6); + db.delete(k2); + assertThat(db.getInsertedKeysCount()).isEqualTo(2); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b3, 3); + assertThat(db.getBlockUpdates().size()).isEqualTo(4); + + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + assertThat(source_db.get(k2).isPresent()).isTrue(); + + // prune block 0 + db.prune(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // prune block 1 + db.prune(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.get(k4).get()).isEqualTo(v4); + // not deleted due to block 2 insert + assertThat(source_db.get(k2).get()).isEqualTo(v3); + + // prune block 2 + db.prune(b2, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).isPresent()).isFalse(); + + // prune block 3 + db.prune(b3, 3); + assertThat(db.getBlockUpdates().size()).isEqualTo(0); + + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + assertThat(source_db.get(k2).isPresent()).isFalse(); + } + + @Test + public void pruningTest_wBatch() { + db.setPruneEnabled(true); + + Map map = new HashMap<>(); + + // block 0 + map.put(k1, v1); + map.put(k2, v2); + map.put(k3, v3); + db.putBatch(map); + + assertThat(db.getInsertedKeysCount()).isEqualTo(3); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + db.storeBlockChanges(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // block 1 + map.clear(); + map.put(k4, v4); + map.put(k2, null); + db.putBatch(map); + + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.get(k2).isPresent()).isTrue(); + assertThat(source_db.get(k4).get()).isEqualTo(v4); + + // block 2 + map.clear(); + map.put(k2, v3); + map.put(k3, null); + db.putBatch(map); + + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b2, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).isPresent()).isTrue(); + + // block 3 + map.clear(); + map.put(k5, v5); + map.put(k6, v6); + map.put(k2, null); + db.putBatch(map); + + assertThat(db.getInsertedKeysCount()).isEqualTo(2); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b3, 3); + assertThat(db.getBlockUpdates().size()).isEqualTo(4); + + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + assertThat(source_db.get(k2).isPresent()).isTrue(); + + // prune block 0 + db.prune(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // prune block 1 + db.prune(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.get(k4).get()).isEqualTo(v4); + // not deleted due to block 2 insert + assertThat(source_db.get(k2).get()).isEqualTo(v3); + + // prune block 2 + db.prune(b2, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).isPresent()).isFalse(); + + // prune block 3 + db.prune(b3, 3); + assertThat(db.getBlockUpdates().size()).isEqualTo(0); + + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + assertThat(source_db.get(k2).isPresent()).isFalse(); + } + + @Test + public void pruningTest_woStoredLevel() { + db.setPruneEnabled(true); + + source_db.put(k2, v2); + source_db.put(k3, v3); + + // block 2 + db.put(k2, v3); + db.delete(k3); + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b2, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).isPresent()).isTrue(); + + // block 3 + db.put(k5, v5); + db.put(k6, v6); + db.delete(k2); + assertThat(db.getInsertedKeysCount()).isEqualTo(2); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b3, 3); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + assertThat(source_db.get(k2).isPresent()).isTrue(); + + // prune block 0 (not stored) + db.prune(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + // prune block 1 (not stored) + db.prune(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + + // prune block 2 + db.prune(b2, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).isPresent()).isFalse(); + + // prune block 3 + db.prune(b3, 3); + assertThat(db.getBlockUpdates().size()).isEqualTo(0); + + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + assertThat(source_db.get(k2).isPresent()).isFalse(); + } + + @Test + public void pruningTest_wFork_onCurrentLevel() { + db.setPruneEnabled(true); + + // block b0 + db.put(k1, v1); + db.put(k2, v2); + db.put(k3, v3); + db.storeBlockChanges(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.keys().size()).isEqualTo(3); + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // block b1 + db.put(k4, v4); + db.put(k1, v2); + db.delete(k2); + db.storeBlockChanges(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.keys().size()).isEqualTo(4); + assertThat(source_db.get(k1).get()).isEqualTo(v2); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k4).get()).isEqualTo(v4); + + // block b2 + db.put(k5, v5); + db.delete(k3); + db.put(k2, v3); + db.put(k1, v4); + db.put(k4, v6); + db.storeBlockChanges(b2, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + + assertThat(source_db.keys().size()).isEqualTo(5); + assertThat(source_db.get(k1).get()).isEqualTo(v4); + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k4).get()).isEqualTo(v6); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + + // block b3 : note same level as block b2 + db.put(k6, v6); + db.delete(k4); + db.put(k2, v4); + db.put(k1, v3); + db.storeBlockChanges(b3, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(4); + + assertThat(source_db.keys().size()).isEqualTo(6); + assertThat(source_db.get(k1).get()).isEqualTo(v3); + assertThat(source_db.get(k2).get()).isEqualTo(v4); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k4).get()).isEqualTo(v6); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + + // prune block b0 + db.prune(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + assertThat(source_db.keys().size()).isEqualTo(6); + + // prune block b1 + db.prune(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + assertThat(source_db.keys().size()).isEqualTo(6); + + // prune block b3 at level 2 (should be called for main chain block) + db.prune(b3, 2); + // also removed the updates for block b2 + assertThat(db.getBlockUpdates().size()).isEqualTo(0); + + assertThat(source_db.keys().size()).isEqualTo(4); + assertThat(source_db.get(k1).get()).isEqualTo(v3); + assertThat(source_db.get(k2).get()).isEqualTo(v4); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k4).isPresent()).isFalse(); + assertThat(source_db.get(k5).isPresent()).isFalse(); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + } + + @Test + public void pruningTest_wFork_onPastLevel() { + db.setPruneEnabled(true); + + // block b0 + db.put(k1, v1); + db.put(k2, v2); + db.put(k3, v3); + db.storeBlockChanges(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.keys().size()).isEqualTo(3); + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // block b1 + db.put(k4, v4); + db.put(k1, v2); + db.delete(k2); + db.storeBlockChanges(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.keys().size()).isEqualTo(4); + assertThat(source_db.get(k1).get()).isEqualTo(v2); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k4).get()).isEqualTo(v4); + + // block b2 : note same level as block b1 + db.put(k5, v5); + db.delete(k3); + db.put(k2, v3); + db.put(k1, v4); + db.storeBlockChanges(b2, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + + assertThat(source_db.keys().size()).isEqualTo(5); + assertThat(source_db.get(k1).get()).isEqualTo(v4); + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k4).get()).isEqualTo(v4); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + + // block b3 + db.put(k6, v6); + db.put(k2, v4); + db.put(k1, v3); + db.storeBlockChanges(b3, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(4); + + assertThat(source_db.keys().size()).isEqualTo(6); + assertThat(source_db.get(k1).get()).isEqualTo(v3); + assertThat(source_db.get(k2).get()).isEqualTo(v4); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k4).get()).isEqualTo(v4); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + + // prune block b0 + db.prune(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + assertThat(source_db.keys().size()).isEqualTo(6); + + // prune block b2 at level 1 : (should be called for main chain block) + db.prune(b2, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + assertThat(source_db.keys().size()).isEqualTo(4); + assertThat(source_db.get(k1).get()).isEqualTo(v3); + assertThat(source_db.get(k2).get()).isEqualTo(v4); + assertThat(source_db.get(k3).isPresent()).isFalse(); + assertThat(source_db.get(k4).isPresent()).isFalse(); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + + // prune block b3 + db.prune(b3, 2); + // also removed the updates for block b2 + assertThat(db.getBlockUpdates().size()).isEqualTo(0); + + assertThat(source_db.keys().size()).isEqualTo(4); + assertThat(source_db.get(k1).get()).isEqualTo(v3); + assertThat(source_db.get(k2).get()).isEqualTo(v4); + assertThat(source_db.get(k3).isPresent()).isFalse(); + assertThat(source_db.get(k4).isPresent()).isFalse(); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + } +} diff --git a/modMcf/test/org/aion/trie/JournalPruneDataSourceTest.java b/modMcf/test/org/aion/trie/JournalPruneDataSourceTest.java deleted file mode 100644 index 83106efb95..0000000000 --- a/modMcf/test/org/aion/trie/JournalPruneDataSourceTest.java +++ /dev/null @@ -1,553 +0,0 @@ -/* ****************************************************************************** - * Copyright (c) 2017-2018 Aion foundation. - * - * This file is part of the aion network project. - * - * The aion network project is free software: you can redistribute it - * and/or modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation, either version 3 of - * the License, or any later version. - * - * The aion network project is distributed in the hope that it will - * be useful, but WITHOUT ANY WARRANTY; without even the implied - * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with the aion network project source files. - * If not, see . - * - * The aion network project leverages useful source code from other - * open source projects. We greatly appreciate the effort that was - * invested in these projects and we thank the individual contributors - * for their work. For provenance information and contributors - * please see . - * - * Contributors to the aion source files in decreasing order of code volume: - * Aion foundation. - ******************************************************************************/ -package org.aion.trie; - -import static com.google.common.truth.Truth.assertThat; - -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import org.aion.base.db.IByteArrayKeyValueDatabase; -import org.aion.db.impl.DatabaseFactory; -import org.aion.log.AionLoggerFactory; -import org.aion.mcf.trie.JournalPruneDataSource; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -/** @author Alexandra Roatis */ -public class JournalPruneDataSourceTest { - - private static final String dbName = "TestDB"; - private static IByteArrayKeyValueDatabase source_db = DatabaseFactory.connect(dbName); - private static JournalPruneDataSource db; - - private static final byte[] k1 = "key1".getBytes(); - private static final byte[] v1 = "value1".getBytes(); - - private static final byte[] k2 = "key2".getBytes(); - private static final byte[] v2 = "value2".getBytes(); - - private static final byte[] k3 = "key3".getBytes(); - private static final byte[] v3 = "value3".getBytes(); - - @BeforeClass - public static void setup() { - // logging to see errors - Map cfg = new HashMap<>(); - cfg.put("DB", "INFO"); - - AionLoggerFactory.init(cfg); - } - - @Before - public void open() { - assertThat(source_db.open()).isTrue(); - db = new JournalPruneDataSource(source_db); - } - - @After - public void close() { - source_db.close(); - assertThat(source_db.isClosed()).isTrue(); - } - - @Test - public void testPut_woPrune() { - db.setPruneEnabled(false); - - assertThat(db.get(k1).isPresent()).isFalse(); - db.put(k1, v1); - assertThat(db.get(k1).get()).isEqualTo(v1); - - // ensure no cached values - assertThat(db.getInsertedKeysCount()).isEqualTo(0); - assertThat(db.getDeletedKeysCount()).isEqualTo(0); - - // ensure the insert was propagated - assertThat(source_db.get(k1).get()).isEqualTo(v1); - } - - @Test - public void testPutBatch_woPrune() { - db.setPruneEnabled(false); - - assertThat(db.get(k1).isPresent()).isFalse(); - assertThat(db.get(k2).isPresent()).isFalse(); - - Map map = new HashMap<>(); - map.put(k1, v1); - map.put(k2, v2); - db.putBatch(map); - - assertThat(v1).isEqualTo(db.get(k1).get()); - assertThat(v2).isEqualTo(db.get(k2).get()); - - // ensure no cached values - assertThat(db.getInsertedKeysCount()).isEqualTo(0); - assertThat(db.getDeletedKeysCount()).isEqualTo(0); - - // ensure the inserts were propagated - assertThat(source_db.get(k1).get()).isEqualTo(v1); - assertThat(source_db.get(k2).get()).isEqualTo(v2); - } - - @Test - public void testUpdate_woPrune() { - db.setPruneEnabled(false); - - // insert - assertThat(db.get(k1).isPresent()).isFalse(); - db.put(k1, v1); - assertThat(db.get(k1).get()).isEqualTo(v1); - assertThat(source_db.get(k1).get()).isEqualTo(v1); - - // update - db.put(k1, v2); - assertThat(db.get(k1).get()).isEqualTo(v2); - assertThat(source_db.get(k1).get()).isEqualTo(v2); - - // indirect delete - db.put(k1, null); - assertThat(db.get(k1).isPresent()).isTrue(); - assertThat(source_db.get(k1).isPresent()).isTrue(); - - // direct delete - db.put(k2, v2); - assertThat(db.get(k2).get()).isEqualTo(v2); - assertThat(source_db.get(k2).get()).isEqualTo(v2); - db.delete(k2); - assertThat(db.get(k2).isPresent()).isTrue(); - assertThat(source_db.get(k2).isPresent()).isTrue(); - - // ensure no cached values - assertThat(db.getDeletedKeysCount()).isEqualTo(0); - assertThat(db.getInsertedKeysCount()).isEqualTo(0); - } - - @Test - public void testUpdateBatch_woPrune() { - db.setPruneEnabled(false); - - // ensure existence - assertThat(db.get(k1).isPresent()).isFalse(); - assertThat(db.get(k2).isPresent()).isFalse(); - assertThat(db.get(k3).isPresent()).isFalse(); - db.put(k1, v1); - db.put(k2, v2); - - assertThat(v1).isEqualTo(db.get(k1).get()); - assertThat(v2).isEqualTo(db.get(k2).get()); - - // check after update - Map ops = new HashMap<>(); - ops.put(k1, null); - ops.put(k2, v1); - ops.put(k3, v3); - db.putBatch(ops); - - assertThat(db.get(k1).isPresent()).isTrue(); - assertThat(db.get(k2).get()).isEqualTo(v1); - assertThat(db.get(k3).get()).isEqualTo(v3); - - assertThat(source_db.get(k1).isPresent()).isTrue(); - assertThat(source_db.get(k2).get()).isEqualTo(v1); - assertThat(source_db.get(k3).get()).isEqualTo(v3); - - // ensure no cached values - assertThat(db.getDeletedKeysCount()).isEqualTo(0); - assertThat(db.getInsertedKeysCount()).isEqualTo(0); - } - - @Test - public void testDelete_woPrune() { - db.setPruneEnabled(false); - - // ensure existence - db.put(k1, v1); - assertThat(db.get(k1).isPresent()).isTrue(); - - // delete not propagated - db.delete(k1); - assertThat(db.get(k1).isPresent()).isTrue(); - assertThat(source_db.get(k1).get()).isEqualTo(v1); - - // ensure no cached values - assertThat(db.getInsertedKeysCount()).isEqualTo(0); - assertThat(db.getDeletedKeysCount()).isEqualTo(0); - } - - @Test - public void testDeleteBatch_woPrune() { - db.setPruneEnabled(false); - - // ensure existence - Map map = new HashMap<>(); - map.put(k1, v1); - map.put(k2, v2); - map.put(k3, null); - db.putBatch(map); - - assertThat(db.get(k1).isPresent()).isTrue(); - assertThat(db.get(k2).isPresent()).isTrue(); - assertThat(db.get(k3).isPresent()).isFalse(); - - // deletes not propagated - db.deleteBatch(map.keySet()); - assertThat(db.get(k1).isPresent()).isTrue(); - assertThat(db.get(k2).isPresent()).isTrue(); - assertThat(db.get(k3).isPresent()).isFalse(); - - // ensure no cached values - assertThat(db.getInsertedKeysCount()).isEqualTo(0); - assertThat(db.getDeletedKeysCount()).isEqualTo(0); - } - - @Test - public void testKeys_woPrune() { - db.setPruneEnabled(false); - - // keys shouldn't be null even when empty - Set keys = db.keys(); - assertThat(db.isEmpty()).isTrue(); - assertThat(keys).isNotNull(); - assertThat(keys.size()).isEqualTo(0); - - // checking after put - db.put(k1, v1); - db.put(k2, v2); - assertThat(db.get(k1).get()).isEqualTo(v1); - assertThat(db.get(k2).get()).isEqualTo(v2); - - keys = db.keys(); - assertThat(keys.size()).isEqualTo(2); - - // checking after delete - db.delete(k2); - assertThat(db.get(k2).isPresent()).isTrue(); - - keys = db.keys(); - assertThat(keys.size()).isEqualTo(2); - - // checking after putBatch - Map ops = new HashMap<>(); - ops.put(k1, null); - ops.put(k2, v2); - ops.put(k3, v3); - db.putBatch(ops); - - keys = db.keys(); - assertThat(keys.size()).isEqualTo(3); - - // checking after deleteBatch - db.deleteBatch(ops.keySet()); - - keys = db.keys(); - assertThat(keys.size()).isEqualTo(3); - - // ensure no cached values - assertThat(db.getInsertedKeysCount()).isEqualTo(0); - assertThat(db.getDeletedKeysCount()).isEqualTo(0); - } - - @Test - public void testIsEmpty_woPrune() { - db.setPruneEnabled(false); - - assertThat(db.isEmpty()).isTrue(); - - // checking after put - db.put(k1, v1); - db.put(k2, v2); - assertThat(db.get(k1).get()).isEqualTo(v1); - assertThat(db.get(k2).get()).isEqualTo(v2); - - assertThat(db.isEmpty()).isFalse(); - - // checking after delete - db.delete(k2); - assertThat(db.get(k2).isPresent()).isTrue(); - assertThat(db.isEmpty()).isFalse(); - db.delete(k1); - assertThat(db.isEmpty()).isFalse(); - - // checking after putBatch - Map ops = new HashMap<>(); - ops.put(k1, null); - ops.put(k2, v2); - ops.put(k3, v3); - db.putBatch(ops); - assertThat(db.isEmpty()).isFalse(); - - // checking after deleteBatch - db.deleteBatch(ops.keySet()); - assertThat(db.isEmpty()).isFalse(); - - // ensure no cached values - assertThat(db.getInsertedKeysCount()).isEqualTo(0); - assertThat(db.getDeletedKeysCount()).isEqualTo(0); - } - - @Test - public void testPut_wPrune() { - db.setPruneEnabled(true); - - assertThat(db.get(k1).isPresent()).isFalse(); - db.put(k1, v1); - assertThat(db.get(k1).get()).isEqualTo(v1); - - // ensure cached values - assertThat(db.getInsertedKeysCount()).isEqualTo(1); - assertThat(db.getDeletedKeysCount()).isEqualTo(0); - - // ensure the insert was propagated - assertThat(source_db.get(k1).get()).isEqualTo(v1); - } - - @Test - public void testPutBatch_wPrune() { - db.setPruneEnabled(true); - - assertThat(db.get(k1).isPresent()).isFalse(); - assertThat(db.get(k2).isPresent()).isFalse(); - - Map map = new HashMap<>(); - map.put(k1, v1); - map.put(k2, v2); - db.putBatch(map); - - assertThat(v1).isEqualTo(db.get(k1).get()); - assertThat(v2).isEqualTo(db.get(k2).get()); - - // ensure cached values - assertThat(db.getInsertedKeysCount()).isEqualTo(2); - assertThat(db.getDeletedKeysCount()).isEqualTo(0); - - // ensure the inserts were propagated - assertThat(source_db.get(k1).get()).isEqualTo(v1); - assertThat(source_db.get(k2).get()).isEqualTo(v2); - } - - @Test - public void testUpdate_wPrune() { - db.setPruneEnabled(true); - - // insert - assertThat(db.get(k1).isPresent()).isFalse(); - db.put(k1, v1); - assertThat(db.get(k1).get()).isEqualTo(v1); - assertThat(source_db.get(k1).get()).isEqualTo(v1); - - // update - db.put(k1, v2); - assertThat(db.get(k1).get()).isEqualTo(v2); - assertThat(source_db.get(k1).get()).isEqualTo(v2); - - // indirect delete - db.put(k1, null); - assertThat(db.get(k1).isPresent()).isTrue(); - assertThat(source_db.get(k1).isPresent()).isTrue(); - - // direct delete - db.put(k2, v2); - assertThat(db.get(k2).get()).isEqualTo(v2); - assertThat(source_db.get(k2).get()).isEqualTo(v2); - db.delete(k2); - assertThat(db.get(k2).isPresent()).isTrue(); - assertThat(source_db.get(k2).isPresent()).isTrue(); - - // ensure cached values - assertThat(db.getDeletedKeysCount()).isEqualTo(2); - assertThat(db.getInsertedKeysCount()).isEqualTo(2); - } - - @Test - public void testUpdateBatch_wPrune() { - db.setPruneEnabled(true); - - // ensure existence - assertThat(db.get(k1).isPresent()).isFalse(); - assertThat(db.get(k2).isPresent()).isFalse(); - assertThat(db.get(k3).isPresent()).isFalse(); - db.put(k1, v1); - db.put(k2, v2); - - assertThat(v1).isEqualTo(db.get(k1).get()); - assertThat(v2).isEqualTo(db.get(k2).get()); - - // check after update - Map ops = new HashMap<>(); - ops.put(k1, null); - ops.put(k2, v1); - ops.put(k3, v3); - db.putBatch(ops); - - assertThat(db.get(k1).isPresent()).isTrue(); - assertThat(db.get(k2).get()).isEqualTo(v1); - assertThat(db.get(k3).get()).isEqualTo(v3); - - assertThat(source_db.get(k1).isPresent()).isTrue(); - assertThat(source_db.get(k2).get()).isEqualTo(v1); - assertThat(source_db.get(k3).get()).isEqualTo(v3); - - // ensure cached values - assertThat(db.getDeletedKeysCount()).isEqualTo(1); - assertThat(db.getInsertedKeysCount()).isEqualTo(3); - } - - @Test - public void testDelete_wPrune() { - db.setPruneEnabled(true); - - // ensure existence - db.put(k1, v1); - assertThat(db.get(k1).isPresent()).isTrue(); - - // delete not propagated - db.delete(k1); - assertThat(db.get(k1).isPresent()).isTrue(); - assertThat(source_db.get(k1).get()).isEqualTo(v1); - - // ensure cached values - assertThat(db.getDeletedKeysCount()).isEqualTo(1); - assertThat(db.getInsertedKeysCount()).isEqualTo(1); - } - - @Test - public void testDeleteBatchWithPrune() { - // ensure existence - Map map = new HashMap<>(); - map.put(k1, v1); - map.put(k2, v2); - map.put(k3, null); - db.putBatch(map); - - assertThat(db.get(k1).isPresent()).isTrue(); - assertThat(db.get(k2).isPresent()).isTrue(); - assertThat(db.get(k3).isPresent()).isFalse(); - - // check presence after delete - db.deleteBatch(map.keySet()); - - // delete operations are delayed till pruning is called - assertThat(db.get(k1).isPresent()).isTrue(); - assertThat(db.get(k2).isPresent()).isTrue(); - assertThat(db.get(k3).isPresent()).isFalse(); - - // ensure cached values - assertThat(db.getDeletedKeysCount()).isEqualTo(3); - assertThat(db.getInsertedKeysCount()).isEqualTo(2); - } - - @Test - public void testKeys() { - db.setPruneEnabled(true); - - // keys shouldn't be null even when empty - Set keys = db.keys(); - assertThat(db.isEmpty()).isTrue(); - assertThat(keys).isNotNull(); - assertThat(keys.size()).isEqualTo(0); - - // checking after put - db.put(k1, v1); - db.put(k2, v2); - assertThat(db.get(k1).get()).isEqualTo(v1); - assertThat(db.get(k2).get()).isEqualTo(v2); - - keys = db.keys(); - assertThat(keys.size()).isEqualTo(2); - - // checking after delete - db.delete(k2); - assertThat(db.get(k2).isPresent()).isTrue(); - - keys = db.keys(); - assertThat(keys.size()).isEqualTo(2); - - // checking after putBatch - Map ops = new HashMap<>(); - ops.put(k1, null); - ops.put(k2, v2); - ops.put(k3, v3); - db.putBatch(ops); - - keys = db.keys(); - assertThat(keys.size()).isEqualTo(3); - - // checking after deleteBatch - db.deleteBatch(ops.keySet()); - - keys = db.keys(); - assertThat(keys.size()).isEqualTo(3); - - // ensure no cached values - assertThat(db.getInsertedKeysCount()).isEqualTo(3); - assertThat(db.getDeletedKeysCount()).isEqualTo(3); - } - - @Test - public void testIsEmpty_wPrune() { - db.setPruneEnabled(true); - - assertThat(db.isEmpty()).isTrue(); - - // checking after put - db.put(k1, v1); - db.put(k2, v2); - assertThat(db.get(k1).get()).isEqualTo(v1); - assertThat(db.get(k2).get()).isEqualTo(v2); - - assertThat(db.isEmpty()).isFalse(); - - // checking after delete - db.delete(k2); - assertThat(db.get(k2).isPresent()).isTrue(); - assertThat(db.isEmpty()).isFalse(); - db.delete(k1); - assertThat(db.isEmpty()).isFalse(); - - // checking after putBatch - Map ops = new HashMap<>(); - ops.put(k1, null); - ops.put(k2, v2); - ops.put(k3, v3); - db.putBatch(ops); - assertThat(db.isEmpty()).isFalse(); - - // checking after deleteBatch - db.deleteBatch(ops.keySet()); - assertThat(db.isEmpty()).isFalse(); - - // ensure no cached values - assertThat(db.getInsertedKeysCount()).isEqualTo(3); - assertThat(db.getDeletedKeysCount()).isEqualTo(3); - } -} diff --git a/modP2p/src/org/aion/p2p/P2pConstant.java b/modP2p/src/org/aion/p2p/P2pConstant.java index 00fa58e911..1584857f9c 100644 --- a/modP2p/src/org/aion/p2p/P2pConstant.java +++ b/modP2p/src/org/aion/p2p/P2pConstant.java @@ -23,5 +23,7 @@ public class P2pConstant { READ_MAX_RATE_TXBC = 20, // write queue timeout - WRITE_MSG_TIMEOUT = 5000; + WRITE_MSG_TIMEOUT = 5000, + + BACKWARD_SYNC_STEP = 128; }