diff --git a/modAionImpl/src/org/aion/zero/impl/db/AbstractRepository.java b/modAionImpl/src/org/aion/zero/impl/db/AbstractRepository.java index f7ef4ec716..9c4e5e2d63 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/AbstractRepository.java +++ b/modAionImpl/src/org/aion/zero/impl/db/AbstractRepository.java @@ -2,8 +2,11 @@ import static org.aion.zero.impl.db.DatabaseUtils.connectAndOpen; import static org.aion.zero.impl.db.DatabaseUtils.verifyAndBuildPath; +import static org.aion.zero.impl.db.DatabaseUtils.verifyDBfileType; import java.io.File; +import java.io.IOException; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -20,6 +23,7 @@ import org.aion.db.store.JournalPruneDataSource; import org.aion.log.AionLoggerFactory; import org.aion.log.LogEnum; +import org.aion.mcf.db.exception.InvalidFileTypeException; import org.aion.zero.impl.config.CfgDb.Names; import org.aion.zero.impl.config.CfgDb.Props; import org.aion.mcf.db.Repository; @@ -112,7 +116,8 @@ public abstract class AbstractRepository implements Repository { * data store cannot be created or opened. * @implNote This function is not locked. Locking must be done from calling function. */ - protected void initializeDatabasesAndCaches() throws InvalidFilePathException { + protected void initializeDatabasesAndCaches() + throws InvalidFilePathException, InvalidFileTypeException, IOException { /* * Given that this function is not in the critical path and only called * on startup, enforce conditions here for safety @@ -132,13 +137,18 @@ protected void initializeDatabasesAndCaches() throws InvalidFilePathException { // } else { DBVendor vendor = DBVendor.fromString(cfg.getDatabaseConfig(Names.DEFAULT).getProperty(Props.DB_TYPE)); + LOGGEN.info("The DB vendor is: {}", vendor); + boolean isPersistent = vendor.isFileBased(); if (isPersistent) { // verify user-provided path File f = new File(this.cfg.getDbPath()); verifyAndBuildPath(f); - } + if (vendor.equals(DBVendor.LEVELDB) || vendor.equals(DBVendor.ROCKSDB)) { + verifyDBfileType(f, vendor.toValue()); + } + } // } // // if (!Arrays.asList(this.cfg.getVendorList()).contains(this.cfg.getActiveVendor())) diff --git a/modAionImpl/src/org/aion/zero/impl/db/DatabaseUtils.java b/modAionImpl/src/org/aion/zero/impl/db/DatabaseUtils.java index 95bc268a9f..1984a5de17 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/DatabaseUtils.java +++ b/modAionImpl/src/org/aion/zero/impl/db/DatabaseUtils.java @@ -8,12 +8,15 @@ import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import java.util.Objects; import java.util.Properties; +import java.util.stream.Stream; import org.aion.db.impl.ByteArrayKeyValueDatabase; import org.aion.db.impl.DatabaseFactory; import org.aion.db.impl.DatabaseFactory.Props; import org.aion.db.impl.PersistenceMethod; import org.aion.mcf.db.exception.InvalidFilePathException; +import org.aion.mcf.db.exception.InvalidFileTypeException; import org.slf4j.Logger; /** @author Alexandra Roatis */ @@ -89,6 +92,84 @@ public static void verifyAndBuildPath(File dbFile) throws InvalidFilePathExcepti } } + /** + * Ensures that the path defined by the dbFile and the fileType in the folders are correct. + * The sst file is part of the file structure of the rocksdb + * The ldb file is part of the file structure of the leveldb + * The OPTIONS file is specific for the rocksdb + * @author Jay Tseng + * @param dbFile the path to be verified. + * @param dbType the database type set by the config file + * @throws InvalidFileTypeException when: + *
    + *
  1. the file type is ldb but found the prefix «OPTIONS» file or .sst file. + *
  2. the file type is sst but found the .ldb file or cannot find the «OPTIONS» file. + *
+ * @throws IOException when: + *
    + *
  1. I/O error for accessing the dbFile. + *
+ */ + static void verifyDBfileType(File dbFile, String dbType) + throws InvalidFileTypeException, IOException { + + // Check if it is a new database folder + if (Objects.requireNonNull(dbFile.listFiles()).length == 0) { + return; + } + + if (dbType.equals("leveldb")) { + try (Stream paths = Files.walk(dbFile.toPath())) { + boolean shouldThrow = + paths.filter(Files::isRegularFile) + .anyMatch( + filepath -> + filepath.getFileName().toString().startsWith("OPTIONS") + || filepath.getFileName().toString().endsWith(".sst")); + if (shouldThrow) { + throw new InvalidFileTypeException( + "Found file type «sst» or the file name prefix «OPTIONS» in the database folder" + + ", it is not matched the current DB settings «" + + dbType + + "» Please check DB settings in .//config/config.xml ."); + } + } + } else if (dbType.equals("rocksdb")) { + try (Stream paths = Files.walk(dbFile.toPath())) { + boolean shouldThrow = + paths.filter(Files::isRegularFile) + .anyMatch(filepath -> + filepath.getFileName().toString().endsWith(".ldb")); + + if (shouldThrow) { + throw new InvalidFileTypeException( + "Found file type «ldb» in the database folder" + + ", it is not matched the current DB settings «" + + dbType + + "» Please check DB settings in .//config/config.xml ."); + } + } + + try (Stream paths = Files.walk(dbFile.toPath())) { + boolean shouldThrow = + paths.filter(Files::isRegularFile) + .noneMatch(filepath -> + filepath.getFileName().toString().startsWith("OPTIONS")); + + if (shouldThrow) { + throw new InvalidFileTypeException( + "Cannot find the file «OPTIONS» in the database folder" + + ", it is not matched with the current DB settings «" + + dbType + + "» Please check DB settings in .//config/config.xml ."); + } + } + } else { + throw new InvalidFileTypeException( + "The db type «" + dbType + "» has not been supported by the kernel."); + } + } + public static boolean deleteRecursively(File file) { Path path = file.toPath(); try { diff --git a/modAionImpl/src/org/aion/zero/impl/db/PendingBlockStore.java b/modAionImpl/src/org/aion/zero/impl/db/PendingBlockStore.java index f03fa66a10..5a608669b8 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/PendingBlockStore.java +++ b/modAionImpl/src/org/aion/zero/impl/db/PendingBlockStore.java @@ -2,10 +2,12 @@ import static org.aion.zero.impl.db.DatabaseUtils.connectAndOpen; import static org.aion.zero.impl.db.DatabaseUtils.verifyAndBuildPath; +import static org.aion.zero.impl.db.DatabaseUtils.verifyDBfileType; import com.google.common.annotations.VisibleForTesting; import java.io.Closeable; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -29,6 +31,7 @@ import org.aion.log.LogEnum; import org.aion.mcf.blockchain.Block; import org.aion.mcf.db.exception.InvalidFilePathException; +import org.aion.mcf.db.exception.InvalidFileTypeException; import org.aion.rlp.RLP; import org.aion.rlp.RLPElement; import org.aion.rlp.RLPList; @@ -92,7 +95,8 @@ public class PendingBlockStore implements Closeable { * @throws InvalidFilePathException when given a persistent database vendor for which the data * store cannot be created or opened. */ - public PendingBlockStore(final Properties _props) throws InvalidFilePathException { + public PendingBlockStore(final Properties _props) + throws InvalidFilePathException, IOException, InvalidFileTypeException { Properties local = new Properties(_props); // check for database persistence requirements @@ -102,6 +106,11 @@ public PendingBlockStore(final Properties _props) throws InvalidFilePathExceptio new File(local.getProperty(Props.DB_PATH), local.getProperty(Props.DB_NAME)); verifyAndBuildPath(pbFolder); + + if (vendor.equals(DBVendor.LEVELDB) || vendor.equals(DBVendor.ROCKSDB)) { + verifyDBfileType(pbFolder, vendor.toValue()); + } + local.setProperty(Props.DB_PATH, pbFolder.getAbsolutePath()); } diff --git a/modAionImpl/test/org/aion/zero/impl/db/PendingBlockStoreTest.java b/modAionImpl/test/org/aion/zero/impl/db/PendingBlockStoreTest.java index ab6308db01..141d77e1f5 100644 --- a/modAionImpl/test/org/aion/zero/impl/db/PendingBlockStoreTest.java +++ b/modAionImpl/test/org/aion/zero/impl/db/PendingBlockStoreTest.java @@ -4,6 +4,7 @@ import static org.aion.zero.impl.db.DatabaseUtils.deleteRecursively; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -15,6 +16,7 @@ import org.aion.log.LogLevel; import org.aion.mcf.blockchain.Block; import org.aion.mcf.db.exception.InvalidFilePathException; +import org.aion.mcf.db.exception.InvalidFileTypeException; import org.aion.util.TestResources; import org.aion.util.types.ByteArrayWrapper; import org.aion.zero.impl.types.A0BlockHeader; @@ -39,7 +41,7 @@ public void testConstructor_wMockDB() { PendingBlockStore pb = null; try { pb = new PendingBlockStore(props); - } catch (InvalidFilePathException e) { + } catch (InvalidFilePathException | IOException | InvalidFileTypeException e) { e.printStackTrace(); } assertThat(pb.isOpen()).isTrue(); @@ -70,7 +72,7 @@ public void testConstructor_wPersistentDB() { PendingBlockStore pb = null; try { pb = new PendingBlockStore(props); - } catch (InvalidFilePathException e) { + } catch (InvalidFilePathException | IOException | InvalidFileTypeException e) { e.printStackTrace(); } assertThat(pb.isOpen()).isTrue(); @@ -97,7 +99,7 @@ public void testConstructor_wPersistentDB() { // check persistence of storage try { pb = new PendingBlockStore(props); - } catch (InvalidFilePathException e) { + } catch (InvalidFilePathException | IOException | InvalidFileTypeException e) { e.printStackTrace(); } assertThat(pb.isOpen()).isTrue(); @@ -119,7 +121,7 @@ public void testAddBlockRange() { PendingBlockStore pb = null; try { pb = new PendingBlockStore(props); - } catch (InvalidFilePathException e) { + } catch (InvalidFilePathException | IOException | InvalidFileTypeException e) { e.printStackTrace(); } assertThat(pb.isOpen()).isTrue(); @@ -193,7 +195,7 @@ public void testAddBlockRange_wException() { PendingBlockStore pb = null; try { pb = new PendingBlockStore(props); - } catch (InvalidFilePathException e) { + } catch (InvalidFilePathException | IOException | InvalidFileTypeException e) { e.printStackTrace(); } assertThat(pb.isOpen()).isTrue(); @@ -215,7 +217,7 @@ public void testLoadBlockRange() { PendingBlockStore pb = null; try { pb = new PendingBlockStore(props); - } catch (InvalidFilePathException e) { + } catch (InvalidFilePathException | IOException | InvalidFileTypeException e) { e.printStackTrace(); } assertThat(pb.isOpen()).isTrue(); @@ -265,7 +267,7 @@ public void testLoadBlockRange_wException() { PendingBlockStore pb = null; try { pb = new PendingBlockStore(props); - } catch (InvalidFilePathException e) { + } catch (InvalidFilePathException | IOException | InvalidFileTypeException e) { e.printStackTrace(); } assertThat(pb.isOpen()).isTrue(); @@ -284,7 +286,7 @@ public void testDropPendingQueues() { PendingBlockStore pb = null; try { pb = new PendingBlockStore(props); - } catch (InvalidFilePathException e) { + } catch (InvalidFilePathException | IOException | InvalidFileTypeException e) { e.printStackTrace(); } assertThat(pb.isOpen()).isTrue(); @@ -325,7 +327,7 @@ public void testDropPendingQueues_wException() { PendingBlockStore pb = null; try { pb = new PendingBlockStore(props); - } catch (InvalidFilePathException e) { + } catch (InvalidFilePathException | IOException | InvalidFileTypeException e) { e.printStackTrace(); } assertThat(pb.isOpen()).isTrue(); @@ -364,7 +366,7 @@ public void testDropPendingQueues_wSingleQueue() { PendingBlockStore pb = null; try { pb = new PendingBlockStore(props); - } catch (InvalidFilePathException e) { + } catch (InvalidFilePathException | IOException | InvalidFileTypeException e) { e.printStackTrace(); } assertThat(pb.isOpen()).isTrue(); @@ -398,4 +400,80 @@ public void testDropPendingQueues_wSingleQueue() { assertThat(pb.getLevelSize()).isEqualTo(1); assertThat(pb.getQueueSize()).isEqualTo(1); } + + @Test(expected = InvalidFileTypeException.class) + public void testSwitchDbVendorleveldbToRocksdbException() + throws InvalidFileTypeException, IOException, InvalidFilePathException { + + File dir = new File(System.getProperty("user.dir"), "tmp-" + System.currentTimeMillis()); + Properties levelDB = new Properties(); + levelDB.setProperty(Props.DB_TYPE, DBVendor.LEVELDB.toValue()); + levelDB.setProperty(Props.DB_PATH, dir.getAbsolutePath()); + levelDB.setProperty(Props.DB_NAME, "pbTest"); + + PendingBlockStore pb; + pb = new PendingBlockStore(levelDB); + assertThat(pb.isOpen()).isTrue(); + + List blocks = TestResources.consecutiveBlocks(16); + assertThat(blocks.size()).isEqualTo(16); + + assertThat(pb.addBlockRange(blocks)).isEqualTo(16); + + pb.close(); + + Properties rocksDB = new Properties(); + rocksDB.setProperty(Props.DB_TYPE, DBVendor.ROCKSDB.toValue()); + rocksDB.setProperty(Props.DB_PATH, dir.getAbsolutePath()); + rocksDB.setProperty(Props.DB_NAME, "pbTest"); + + try { + pb = new PendingBlockStore(rocksDB); + } catch (Exception e) { + assertThat(deleteRecursively(dir)).isTrue(); + throw e; + } + + // This test should not reach to here! + assertThat(pb.isOpen()).isTrue(); + pb.close(); + } + + @Test(expected = InvalidFileTypeException.class) + public void testSwitchDbVendorRocksdbToLeveldbException() + throws InvalidFileTypeException, IOException, InvalidFilePathException { + + File dir = new File(System.getProperty("user.dir"), "tmp-" + System.currentTimeMillis()); + Properties rocksDB = new Properties(); + rocksDB.setProperty(Props.DB_TYPE, DBVendor.ROCKSDB.toValue()); + rocksDB.setProperty(Props.DB_PATH, dir.getAbsolutePath()); + rocksDB.setProperty(Props.DB_NAME, "pbTest"); + + PendingBlockStore pb; + pb = new PendingBlockStore(rocksDB); + assertThat(pb.isOpen()).isTrue(); + + List blocks = TestResources.consecutiveBlocks(16); + assertThat(blocks.size()).isEqualTo(16); + + assertThat(pb.addBlockRange(blocks)).isEqualTo(16); + + pb.close(); + + Properties levelDB = new Properties(); + levelDB.setProperty(Props.DB_TYPE, DBVendor.LEVELDB.toValue()); + levelDB.setProperty(Props.DB_PATH, dir.getAbsolutePath()); + levelDB.setProperty(Props.DB_NAME, "pbTest"); + + try { + pb = new PendingBlockStore(levelDB); + } catch (Exception e) { + assertThat(deleteRecursively(dir)).isTrue(); + throw e; + } + + // This test should not reach to here! + assertThat(pb.isOpen()).isTrue(); + pb.close(); + } } diff --git a/modMcf/src/org/aion/mcf/db/exception/InvalidFileTypeException.java b/modMcf/src/org/aion/mcf/db/exception/InvalidFileTypeException.java new file mode 100644 index 0000000000..05169cc981 --- /dev/null +++ b/modMcf/src/org/aion/mcf/db/exception/InvalidFileTypeException.java @@ -0,0 +1,15 @@ +package org.aion.mcf.db.exception; + +/** Invalid file path exception. */ +public class InvalidFileTypeException extends Exception { + + private static final long serialVersionUID = -7022793792489250954L; + + public InvalidFileTypeException() { + super(); + } + + public InvalidFileTypeException(String message) { + super(message); + } +} diff --git a/networks/amity/config.xml b/networks/amity/config.xml index f7cec0982b..35d7ab15af 100755 --- a/networks/amity/config.xml +++ b/networks/amity/config.xml @@ -66,9 +66,9 @@ FULL - leveldb + rocksdb - false + true diff --git a/networks/custom/config.xml b/networks/custom/config.xml index 5b9d986e59..910f483094 100644 --- a/networks/custom/config.xml +++ b/networks/custom/config.xml @@ -67,9 +67,9 @@ FULL - leveldb + rocksdb - false + true diff --git a/networks/mainnet/config.xml b/networks/mainnet/config.xml index 456e4b9c9e..bbf0ad2164 100644 --- a/networks/mainnet/config.xml +++ b/networks/mainnet/config.xml @@ -73,9 +73,9 @@ FULL - leveldb + rocksdb - false + true