block import latency enhancement

modAion/src/org/aion/zero/db/AionContractDetailsImpl.java

@@ -28,7 +28,6 @@
 import static org.aion.crypto.HashUtil.EMPTY_TRIE_HASH;
 import static org.aion.crypto.HashUtil.h256;
 
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
@@ -149,13 +148,33 @@ public byte[] getStorageHash() {
      */
     @Override
     public void decode(byte[] rlpCode) {
+        decode(rlpCode, false);
+    }
+
+    /**
+     * Decodes an AionContractDetailsImpl object from the RLP encoding rlpCode with fast check does
+     * the contractDetails needs external storage.
+     *
+     * @param rlpCode The encoding to decode.
+     * @param fastCheck set fastCheck option.
+     */
+    @Override
+    public void decode(byte[] rlpCode, boolean fastCheck) {
         RLPList data = RLP.decode2(rlpCode);
         RLPList rlpList = (RLPList) data.get(0);
 
-        RLPItem address = (RLPItem) rlpList.get(0);
         RLPItem isExternalStorage = (RLPItem) rlpList.get(1);
-        RLPItem storageRoot = (RLPItem) rlpList.get(2);
         RLPItem storage = (RLPItem) rlpList.get(3);
+        this.externalStorage = isExternalStorage.getRLPData().length > 0;
+        boolean keepStorageInMem = storage.getRLPData().length <= detailsInMemoryStorageLimit;
+
+        // No externalStorage require.
+        if (fastCheck && !externalStorage && keepStorageInMem) {
+            return;
+        }
+
+        RLPItem address = (RLPItem) rlpList.get(0);
+        RLPItem storageRoot = (RLPItem) rlpList.get(2);
         RLPElement code = rlpList.get(4);
 
         if (address.getRLPData() == null) {
@@ -173,7 +192,6 @@ public void decode(byte[] rlpCode) {
         }
 
         // load/deserialize storage trie
-        this.externalStorage = !Arrays.equals(isExternalStorage.getRLPData(), EMPTY_BYTE_ARRAY);
         if (externalStorage) {
             storageTrie = new SecureTrie(getExternalStorageDataSource(), storageRoot.getRLPData());
         } else {
@@ -182,7 +200,7 @@ public void decode(byte[] rlpCode) {
         storageTrie.withPruningEnabled(prune > 0);
 
         // switch from in-memory to external storage
-        if (!externalStorage && storage.getRLPData().length > detailsInMemoryStorageLimit) {
+        if (!externalStorage && !keepStorageInMem) {
             externalStorage = true;
             storageTrie.getCache().setDB(getExternalStorageDataSource());
         }

modAionBase/src/org/aion/base/db/IContractDetails.java

@@ -100,6 +100,16 @@ public interface IContractDetails<DW> {
      */
     void decode(byte[] rlpCode);
 
+    /**
+     * Decodes an IContractDetails object from the RLP encoding rlpCode including the fast check
+     * optional.
+     *
+     * @implNote Implementing classes may not necessarily support this method.
+     * @param rlpCode The encoding to decode.
+     * @param fastCheck fast check does the contractDetails needs syncing with external storage
+     */
+    void decode(byte[] rlpCode, boolean fastCheck);
+
     /**
      * Sets the dirty value to dirty.
      *

modAionBase/src/org/aion/base/db/IKeyValueStore.java

@@ -36,9 +36,9 @@
 package org.aion.base.db;
 
 import java.util.Collection;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Optional;
-import java.util.Set;
 
 /**
  * Functionality for a key-value cache allowing itemized updates.
@@ -63,13 +63,15 @@ public interface IKeyValueStore<K, V> extends AutoCloseable {
     boolean isEmpty();
 
     /**
-     * Returns the set of keys for the database.
+     * Returns an {@link Iterator} over the set of keys stored in the database at the time when the
+     * keys were requested. A snapshot can be used to ensure that the entries do not change while
+     * iterating through the keys.
      *
-     * @return Set of keys
+     * @return an iterator over the set of stored keys
      * @throws RuntimeException if the data store is closed
-     * @apiNote Returns an empty set if the database keys could not be retrieved.
+     * @apiNote Returns an empty iterator if the database keys could not be retrieved.
      */
-    Set<K> keys();
+    Iterator<K> keys();
 
     /**
      * get retrieves a value from the database, returning an optional, it is fulfilled if a value

modAionImpl/src/org/aion/zero/impl/Version.java

@@ -24,7 +24,7 @@
 package org.aion.zero.impl;
 
 public class Version {
-    public static final String KERNEL_VERSION = "0.3.2";
+    public static final String KERNEL_VERSION = "0.3.2.2";
     public static final String REPO_VERSION = "0.1.0";
     public static final boolean FORK = true;
 }

modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java

@@ -32,6 +32,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -332,8 +333,9 @@ public List<byte[]> getPoolTx() {
         List<byte[]> rtn = new ArrayList<>();
         rwLock.readLock().lock();
         try {
-            Set<byte[]> keySet = txPoolDatabase.keys();
-            for (byte[] b : keySet) {
+            Iterator<byte[]> iterator = txPoolDatabase.keys();
+            while (iterator.hasNext()) {
+                byte[] b = iterator.next();
                 if (txPoolDatabase.get(b).isPresent()) {
                     rtn.add(txPoolDatabase.get(b).get());
                 }
@@ -351,8 +353,9 @@ public List<byte[]> getCacheTx() {
         List<byte[]> rtn = new ArrayList<>();
         rwLock.readLock().lock();
         try {
-            Set<byte[]> keySet = pendingTxCacheDatabase.keys();
-            for (byte[] b : keySet) {
+            Iterator<byte[]> iterator = pendingTxCacheDatabase.keys();
+            while (iterator.hasNext()) {
+                byte[] b = iterator.next();
                 if (pendingTxCacheDatabase.get(b).isPresent()) {
                     rtn.add(pendingTxCacheDatabase.get(b).get());
                 }

modAionImpl/src/org/aion/zero/impl/db/PendingBlockStore.java

@@ -36,6 +36,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -519,7 +520,7 @@ private int addBlockRange(AionBlock first, List<AionBlock> blockRange) {
     int getIndexSize() {
         databaseLock.readLock().lock();
         try {
-            return indexSource.keys().size();
+            return countDatabaseKeys(indexSource);
         } finally {
             databaseLock.readLock().unlock();
         }
@@ -533,7 +534,7 @@ int getIndexSize() {
     int getLevelSize() {
         databaseLock.readLock().lock();
         try {
-            return levelDatabase.keys().size();
+            return countDatabaseKeys(levelDatabase);
         } finally {
             databaseLock.readLock().unlock();
         }
@@ -547,12 +548,22 @@ int getLevelSize() {
     int getQueueSize() {
         databaseLock.readLock().lock();
         try {
-            return queueDatabase.keys().size();
+            return countDatabaseKeys(queueDatabase);
         } finally {
             databaseLock.readLock().unlock();
         }
     }
 
+    private static int countDatabaseKeys(IByteArrayKeyValueDatabase db) {
+        int size = 0;
+        Iterator<byte[]> iterator = db.keys();
+        while (iterator.hasNext()) {
+            iterator.next();
+            size++;
+        }
+        return size;
+    }
+
     /**
      * Retrieves blocks from storage based on the height of the first block in the range.
      *

modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java

@@ -592,7 +592,7 @@ private ImportResult importBlock(AionBlock b, String displayId, PeerState state)
         // 1 sec import time and more than 10 min since last compact
         if (t2 - t1 > SLOW_IMPORT_TIME && t2 - lastCompactTime > COMPACT_FREQUENCY) {
             t1 = System.currentTimeMillis();
-            this.chain.compactState();
+            //this.chain.compactState();
             t2 = System.currentTimeMillis();
             log.info("Compacting state database due to slow IO time. Completed in {} ms.", t2 - t1);
             lastCompactTime = t2;

modAionImpl/test/org/aion/zero/impl/BlockchainDataRecoveryTest.java

@@ -26,6 +26,7 @@
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import org.aion.base.db.IByteArrayKeyValueDatabase;
@@ -263,7 +264,10 @@ public void testRecoverWorldStateWithoutGenesis() {
 
         repo.flush();
         List<byte[]> statesToDelete = new ArrayList<>();
-        statesToDelete.addAll(database.keys());
+        Iterator<byte[]> iterator = database.keys();
+        while (iterator.hasNext()) {
+            statesToDelete.add(iterator.next());
+        }
 
         for (byte[] key : statesToDelete) {
             database.delete(key);

modDbImpl/src/org/aion/db/generic/CacheIteratorWrapper.java

@@ -0,0 +1,76 @@
+package org.aion.db.generic;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import org.aion.base.util.ByteArrayWrapper;
+
+/**
+ * A wrapper for the iterator needed by {@link DatabaseWithCache} conforming to the {@link Iterator}
+ * interface.
+ *
+ * @implNote Assumes that the given database iterator does not return duplicate values.
+ * @author Alexandra Roatis
+ */
+public class CacheIteratorWrapper implements Iterator<byte[]> {
+    private final Iterator<byte[]> iterator;
+    private byte[] next;
+    private final List<ByteArrayWrapper> additions;
+    private final List<ByteArrayWrapper> removals;
+
+    /**
+     * @implNote Building two wrappers for the same {@link Iterator} will lead to inconsistent
+     *     behavior.
+     */
+    public CacheIteratorWrapper(
+            final Iterator<byte[]> iterator, Map<ByteArrayWrapper, byte[]> dirtyEntries) {
+        this.iterator = iterator;
+        additions = new ArrayList<>();
+        removals = new ArrayList<>();
+
+        for (Map.Entry<ByteArrayWrapper, byte[]> entry : dirtyEntries.entrySet()) {
+            if (entry.getValue() == null) {
+                removals.add(entry.getKey());
+            } else {
+                additions.add(entry.getKey());
+            }
+        }
+    }
+
+    @Override
+    public boolean hasNext() {
+        boolean seek = true;
+        ByteArrayWrapper wrapper;
+        // check in the database iterator
+        while (seek && iterator.hasNext()) {
+            next = iterator.next();
+            wrapper = ByteArrayWrapper.wrap(next);
+            if (removals.contains(wrapper)) {
+                // key deleted, move to next in iterator
+                removals.remove(wrapper);
+            } else if (additions.contains(wrapper)) {
+                // found an entry that was updated
+                seek = false;
+                additions.remove(wrapper);
+            } else {
+                // found an entry that was not changed
+                seek = false;
+            }
+        }
+
+        // exhausted the initial iterator, trying the dirty entries
+        // check in the cached entries
+        if (seek && !additions.isEmpty()) {
+            next = additions.remove(0).getData();
+            seek = false;
+        }
+
+        return !seek;
+    }
+
+    @Override
+    public byte[] next() {
+        return next;
+    }
+}

modDbImpl/src/org/aion/db/generic/DatabaseWithCache.java

@@ -41,10 +41,9 @@
 import com.google.common.primitives.Longs;
 import java.util.Collection;
 import java.util.HashMap;
-import java.util.HashSet;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Optional;
-import java.util.Set;
 import org.aion.base.db.IByteArrayKeyValueDatabase;
 import org.aion.base.util.ByteArrayWrapper;
 import org.aion.db.impl.AbstractDB;
@@ -378,26 +377,9 @@ public boolean isEmpty() {
     }
 
     @Override
-    public Set<byte[]> keys() {
-
-        Set<byte[]> keys = new HashSet<>();
-
+    public Iterator<byte[]> keys() {
         check();
-
-        // add all database keys
-        keys.addAll(database.keys());
-
-        // add updated cached keys
-        dirtyEntries.forEach(
-                (k, v) -> {
-                    if (v == null) {
-                        keys.remove(k.getData());
-                    } else {
-                        keys.add(k.getData());
-                    }
-                });
-
-        return keys;
+        return new CacheIteratorWrapper(database.keys(), dirtyEntries);
     }
 
     /**

modDbImpl/src/org/aion/db/generic/LockedDatabase.java

@@ -24,9 +24,9 @@
 package org.aion.db.generic;
 
 import java.util.Collection;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Optional;
-import java.util.Set;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import org.aion.base.db.IByteArrayKeyValueDatabase;
@@ -220,7 +220,7 @@ public boolean isEmpty() {
     }
 
     @Override
-    public Set<byte[]> keys() {
+    public Iterator<byte[]> keys() {
         // acquire read lock
         lock.readLock().lock();
 

modDbImpl/src/org/aion/db/generic/TimedDatabase.java

@@ -24,9 +24,9 @@
 package org.aion.db.generic;
 
 import java.util.Collection;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Optional;
-import java.util.Set;
 import org.aion.base.db.IByteArrayKeyValueDatabase;
 import org.aion.base.util.Hex;
 import org.aion.log.AionLoggerFactory;
@@ -174,9 +174,9 @@ public boolean isEmpty() {
     }
 
     @Override
-    public Set<byte[]> keys() {
+    public Iterator<byte[]> keys() {
         long t1 = System.nanoTime();
-        Set<byte[]> result = database.keys();
+        Iterator<byte[]> result = database.keys();
         long t2 = System.nanoTime();
 
         LOG.debug(database.toString() + " keys() in " + (t2 - t1) + " ns.");