AKI-646 switch default DB to the rocksDB

modAionImpl/src/org/aion/zero/impl/db/AbstractRepository.java

@@ -2,8 +2,11 @@
 
 import static org.aion.zero.impl.db.DatabaseUtils.connectAndOpen;
 import static org.aion.zero.impl.db.DatabaseUtils.verifyAndBuildPath;
+import static org.aion.zero.impl.db.DatabaseUtils.verifyDBfileType;
 
 import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -20,6 +23,7 @@
 import org.aion.db.store.JournalPruneDataSource;
 import org.aion.log.AionLoggerFactory;
 import org.aion.log.LogEnum;
+import org.aion.mcf.db.exception.InvalidFileTypeException;
 import org.aion.zero.impl.config.CfgDb.Names;
 import org.aion.zero.impl.config.CfgDb.Props;
 import org.aion.mcf.db.Repository;
@@ -112,7 +116,8 @@ public abstract class AbstractRepository implements Repository<AccountState> {
      *     data store cannot be created or opened.
      * @implNote This function is not locked. Locking must be done from calling function.
      */
-    protected void initializeDatabasesAndCaches() throws InvalidFilePathException {
+    protected void initializeDatabasesAndCaches()
+        throws InvalidFilePathException, InvalidFileTypeException, IOException {
         /*
          * Given that this function is not in the critical path and only called
          * on startup, enforce conditions here for safety
@@ -132,13 +137,18 @@ protected void initializeDatabasesAndCaches() throws InvalidFilePathException {
         //        } else {
 
         DBVendor vendor = DBVendor.fromString(cfg.getDatabaseConfig(Names.DEFAULT).getProperty(Props.DB_TYPE));
+        LOGGEN.info("The DB vendor is: {}", vendor);
+
         boolean isPersistent = vendor.isFileBased();
         if (isPersistent) {
             // verify user-provided path
             File f = new File(this.cfg.getDbPath());
             verifyAndBuildPath(f);
-        }
 
+            if (vendor.equals(DBVendor.LEVELDB) || vendor.equals(DBVendor.ROCKSDB)) {
+                verifyDBfileType(f, vendor.toValue());
+            }
+        }
         //        }
         //
         //        if (!Arrays.asList(this.cfg.getVendorList()).contains(this.cfg.getActiveVendor()))

modAionImpl/src/org/aion/zero/impl/db/DatabaseUtils.java

@@ -8,12 +8,15 @@
 import java.nio.file.Paths;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Objects;
 import java.util.Properties;
+import java.util.stream.Stream;
 import org.aion.db.impl.ByteArrayKeyValueDatabase;
 import org.aion.db.impl.DatabaseFactory;
 import org.aion.db.impl.DatabaseFactory.Props;
 import org.aion.db.impl.PersistenceMethod;
 import org.aion.mcf.db.exception.InvalidFilePathException;
+import org.aion.mcf.db.exception.InvalidFileTypeException;
 import org.slf4j.Logger;
 
 /** @author Alexandra Roatis */
@@ -89,6 +92,84 @@ public static void verifyAndBuildPath(File dbFile) throws InvalidFilePathExcepti
         }
     }
 
+    /**
+     * Ensures that the path defined by the dbFile and the fileType in the folders are correct.
+     * The sst file is part of the file structure of the rocksdb
+     * The ldb file is part of the file structure of the leveldb
+     * The OPTIONS file is specific for the rocksdb
+     * @author Jay Tseng
+     * @param dbFile the path to be verified.
+     * @param dbType the database type set by the config file
+     * @throws InvalidFileTypeException when:
+     *     <ol>
+     *       <li>the file type is ldb but found the prefix «OPTIONS» file or .sst file.
+     *       <li>the file type is sst but found the .ldb file or cannot find the «OPTIONS» file.
+     *     </ol>
+     * @throws IOException when:
+     *     <ol>
+     *       <li> I/O error for accessing the dbFile.
+     *     </ol>
+     */
+    static void verifyDBfileType(File dbFile, String dbType)
+        throws InvalidFileTypeException, IOException {
+
+        // Check if it is a new database folder
+        if (Objects.requireNonNull(dbFile.listFiles()).length == 0) {
+            return;
+        }
+
+        if (dbType.equals("leveldb")) {
+            try (Stream<Path> paths = Files.walk(dbFile.toPath())) {
+                boolean shouldThrow =
+                    paths.filter(Files::isRegularFile)
+                        .anyMatch(
+                            filepath ->
+                                filepath.getFileName().toString().startsWith("OPTIONS")
+                                    || filepath.getFileName().toString().endsWith(".sst"));
+                if (shouldThrow) {
+                    throw new InvalidFileTypeException(
+                        "Found file type «sst» or the file name prefix «OPTIONS» in the database folder"
+                            + ", it is not matched the current DB settings «"
+                            + dbType
+                            + "» Please check DB settings in ./<network>/config/config.xml .");
+                }
+            }
+        } else if (dbType.equals("rocksdb")) {
+            try (Stream<Path> paths = Files.walk(dbFile.toPath())) {
+                boolean shouldThrow =
+                    paths.filter(Files::isRegularFile)
+                        .anyMatch(filepath ->
+                            filepath.getFileName().toString().endsWith(".ldb"));
+
+                if (shouldThrow) {
+                    throw new InvalidFileTypeException(
+                        "Found file type «ldb» in the database folder"
+                            + ", it is not matched the current DB settings «"
+                            + dbType
+                            + "» Please check DB settings in ./<network>/config/config.xml .");
+                }
+            }
+
+            try (Stream<Path> paths = Files.walk(dbFile.toPath())) {
+                boolean shouldThrow =
+                    paths.filter(Files::isRegularFile)
+                        .noneMatch(filepath ->
+                            filepath.getFileName().toString().startsWith("OPTIONS"));
+
+                if (shouldThrow) {
+                    throw new InvalidFileTypeException(
+                        "Cannot find the file «OPTIONS» in the database folder"
+                            + ", it is not matched with the current DB settings «"
+                            + dbType
+                            + "» Please check DB settings in ./<network>/config/config.xml .");
+                }
+            }
+        } else {
+            throw new InvalidFileTypeException(
+                    "The db type «" + dbType + "» has not been supported by the kernel.");
+        }
+    }
+
     public static boolean deleteRecursively(File file) {
         Path path = file.toPath();
         try {

modAionImpl/src/org/aion/zero/impl/db/PendingBlockStore.java

@@ -2,10 +2,12 @@
 
 import static org.aion.zero.impl.db.DatabaseUtils.connectAndOpen;
 import static org.aion.zero.impl.db.DatabaseUtils.verifyAndBuildPath;
+import static org.aion.zero.impl.db.DatabaseUtils.verifyDBfileType;
 
 import com.google.common.annotations.VisibleForTesting;
 import java.io.Closeable;
 import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -29,6 +31,7 @@
 import org.aion.log.LogEnum;
 import org.aion.mcf.blockchain.Block;
 import org.aion.mcf.db.exception.InvalidFilePathException;
+import org.aion.mcf.db.exception.InvalidFileTypeException;
 import org.aion.rlp.RLP;
 import org.aion.rlp.RLPElement;
 import org.aion.rlp.RLPList;
@@ -92,7 +95,8 @@ public class PendingBlockStore implements Closeable {
      * @throws InvalidFilePathException when given a persistent database vendor for which the data
      *     store cannot be created or opened.
      */
-    public PendingBlockStore(final Properties _props) throws InvalidFilePathException {
+    public PendingBlockStore(final Properties _props)
+        throws InvalidFilePathException, IOException, InvalidFileTypeException {
         Properties local = new Properties(_props);
 
         // check for database persistence requirements
@@ -102,6 +106,11 @@ public PendingBlockStore(final Properties _props) throws InvalidFilePathExceptio
                     new File(local.getProperty(Props.DB_PATH), local.getProperty(Props.DB_NAME));
 
             verifyAndBuildPath(pbFolder);
+
+            if (vendor.equals(DBVendor.LEVELDB) || vendor.equals(DBVendor.ROCKSDB)) {
+                verifyDBfileType(pbFolder, vendor.toValue());
+            }
+
             local.setProperty(Props.DB_PATH, pbFolder.getAbsolutePath());
         }
 

modAionImpl/test/org/aion/zero/impl/db/PendingBlockStoreTest.java

@@ -4,6 +4,7 @@
 import static org.aion.zero.impl.db.DatabaseUtils.deleteRecursively;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -15,6 +16,7 @@
 import org.aion.log.LogLevel;
 import org.aion.mcf.blockchain.Block;
 import org.aion.mcf.db.exception.InvalidFilePathException;
+import org.aion.mcf.db.exception.InvalidFileTypeException;
 import org.aion.util.TestResources;
 import org.aion.util.types.ByteArrayWrapper;
 import org.aion.zero.impl.types.A0BlockHeader;
@@ -39,7 +41,7 @@ public void testConstructor_wMockDB() {
         PendingBlockStore pb = null;
         try {
             pb = new PendingBlockStore(props);
-        } catch (InvalidFilePathException e) {
+        } catch (InvalidFilePathException | IOException | InvalidFileTypeException e) {
             e.printStackTrace();
         }
         assertThat(pb.isOpen()).isTrue();
@@ -70,7 +72,7 @@ public void testConstructor_wPersistentDB() {
         PendingBlockStore pb = null;
         try {
             pb = new PendingBlockStore(props);
-        } catch (InvalidFilePathException e) {
+        } catch (InvalidFilePathException | IOException | InvalidFileTypeException e) {
             e.printStackTrace();
         }
         assertThat(pb.isOpen()).isTrue();
@@ -97,7 +99,7 @@ public void testConstructor_wPersistentDB() {
         // check persistence of storage
         try {
             pb = new PendingBlockStore(props);
-        } catch (InvalidFilePathException e) {
+        } catch (InvalidFilePathException | IOException | InvalidFileTypeException e) {
             e.printStackTrace();
         }
         assertThat(pb.isOpen()).isTrue();
@@ -119,7 +121,7 @@ public void testAddBlockRange() {
         PendingBlockStore pb = null;
         try {
             pb = new PendingBlockStore(props);
-        } catch (InvalidFilePathException e) {
+        } catch (InvalidFilePathException | IOException | InvalidFileTypeException e) {
             e.printStackTrace();
         }
         assertThat(pb.isOpen()).isTrue();
@@ -193,7 +195,7 @@ public void testAddBlockRange_wException() {
         PendingBlockStore pb = null;
         try {
             pb = new PendingBlockStore(props);
-        } catch (InvalidFilePathException e) {
+        } catch (InvalidFilePathException | IOException | InvalidFileTypeException e) {
             e.printStackTrace();
         }
         assertThat(pb.isOpen()).isTrue();
@@ -215,7 +217,7 @@ public void testLoadBlockRange() {
         PendingBlockStore pb = null;
         try {
             pb = new PendingBlockStore(props);
-        } catch (InvalidFilePathException e) {
+        } catch (InvalidFilePathException | IOException | InvalidFileTypeException e) {
             e.printStackTrace();
         }
         assertThat(pb.isOpen()).isTrue();
@@ -265,7 +267,7 @@ public void testLoadBlockRange_wException() {
         PendingBlockStore pb = null;
         try {
             pb = new PendingBlockStore(props);
-        } catch (InvalidFilePathException e) {
+        } catch (InvalidFilePathException | IOException | InvalidFileTypeException e) {
             e.printStackTrace();
         }
         assertThat(pb.isOpen()).isTrue();
@@ -284,7 +286,7 @@ public void testDropPendingQueues() {
         PendingBlockStore pb = null;
         try {
             pb = new PendingBlockStore(props);
-        } catch (InvalidFilePathException e) {
+        } catch (InvalidFilePathException | IOException | InvalidFileTypeException e) {
             e.printStackTrace();
         }
         assertThat(pb.isOpen()).isTrue();
@@ -325,7 +327,7 @@ public void testDropPendingQueues_wException() {
         PendingBlockStore pb = null;
         try {
             pb = new PendingBlockStore(props);
-        } catch (InvalidFilePathException e) {
+        } catch (InvalidFilePathException | IOException | InvalidFileTypeException e) {
             e.printStackTrace();
         }
         assertThat(pb.isOpen()).isTrue();
@@ -364,7 +366,7 @@ public void testDropPendingQueues_wSingleQueue() {
         PendingBlockStore pb = null;
         try {
             pb = new PendingBlockStore(props);
-        } catch (InvalidFilePathException e) {
+        } catch (InvalidFilePathException | IOException | InvalidFileTypeException e) {
             e.printStackTrace();
         }
         assertThat(pb.isOpen()).isTrue();
@@ -398,4 +400,80 @@ public void testDropPendingQueues_wSingleQueue() {
         assertThat(pb.getLevelSize()).isEqualTo(1);
         assertThat(pb.getQueueSize()).isEqualTo(1);
     }
+
+    @Test(expected = InvalidFileTypeException.class)
+    public void testSwitchDbVendorleveldbToRocksdbException()
+        throws InvalidFileTypeException, IOException, InvalidFilePathException {
+
+        File dir = new File(System.getProperty("user.dir"), "tmp-" + System.currentTimeMillis());
+        Properties levelDB = new Properties();
+        levelDB.setProperty(Props.DB_TYPE, DBVendor.LEVELDB.toValue());
+        levelDB.setProperty(Props.DB_PATH, dir.getAbsolutePath());
+        levelDB.setProperty(Props.DB_NAME, "pbTest");
+
+        PendingBlockStore pb;
+        pb = new PendingBlockStore(levelDB);
+        assertThat(pb.isOpen()).isTrue();
+
+        List<Block> blocks = TestResources.consecutiveBlocks(16);
+        assertThat(blocks.size()).isEqualTo(16);
+
+        assertThat(pb.addBlockRange(blocks)).isEqualTo(16);
+
+        pb.close();
+
+        Properties rocksDB = new Properties();
+        rocksDB.setProperty(Props.DB_TYPE, DBVendor.ROCKSDB.toValue());
+        rocksDB.setProperty(Props.DB_PATH, dir.getAbsolutePath());
+        rocksDB.setProperty(Props.DB_NAME, "pbTest");
+
+        try {
+            pb = new PendingBlockStore(rocksDB);
+        } catch (Exception e) {
+            assertThat(deleteRecursively(dir)).isTrue();
+            throw e;
+        }
+
+        // This test should not reach to here!
+        assertThat(pb.isOpen()).isTrue();
+        pb.close();
+    }
+
+    @Test(expected = InvalidFileTypeException.class)
+    public void testSwitchDbVendorRocksdbToLeveldbException()
+        throws InvalidFileTypeException, IOException, InvalidFilePathException {
+
+        File dir = new File(System.getProperty("user.dir"), "tmp-" + System.currentTimeMillis());
+        Properties rocksDB = new Properties();
+        rocksDB.setProperty(Props.DB_TYPE, DBVendor.ROCKSDB.toValue());
+        rocksDB.setProperty(Props.DB_PATH, dir.getAbsolutePath());
+        rocksDB.setProperty(Props.DB_NAME, "pbTest");
+
+        PendingBlockStore pb;
+        pb = new PendingBlockStore(rocksDB);
+        assertThat(pb.isOpen()).isTrue();
+
+        List<Block> blocks = TestResources.consecutiveBlocks(16);
+        assertThat(blocks.size()).isEqualTo(16);
+
+        assertThat(pb.addBlockRange(blocks)).isEqualTo(16);
+
+        pb.close();
+
+        Properties levelDB = new Properties();
+        levelDB.setProperty(Props.DB_TYPE, DBVendor.LEVELDB.toValue());
+        levelDB.setProperty(Props.DB_PATH, dir.getAbsolutePath());
+        levelDB.setProperty(Props.DB_NAME, "pbTest");
+
+        try {
+            pb = new PendingBlockStore(levelDB);
+        } catch (Exception e) {
+            assertThat(deleteRecursively(dir)).isTrue();
+            throw e;
+        }
+
+        // This test should not reach to here!
+        assertThat(pb.isOpen()).isTrue();
+        pb.close();
+    }
 }

modMcf/src/org/aion/mcf/db/exception/InvalidFileTypeException.java

@@ -0,0 +1,15 @@
+package org.aion.mcf.db.exception;
+
+/** Invalid file path exception. */
+public class InvalidFileTypeException extends Exception {
+
+    private static final long serialVersionUID = -7022793792489250954L;
+
+    public InvalidFileTypeException() {
+        super();
+    }
+
+    public InvalidFileTypeException(String message) {
+        super(message);
+    }
+}