diff --git a/modAionImpl/build.gradle b/modAionImpl/build.gradle index ef1c3b485e..126fdf85ec 100644 --- a/modAionImpl/build.gradle +++ b/modAionImpl/build.gradle @@ -42,6 +42,7 @@ dependencies { testCompile files('../lib/org-aion-avm-api.jar') + testCompile 'network.aion:crypto4j:0.4.0' testCompile 'junit:junit:4.12' testCompile 'pl.pragmatists:JUnitParams:1.1.1' testCompile 'org.hamcrest:hamcrest-all:1.3' diff --git a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java index 6206061a71..8d910a2df6 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java +++ b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java @@ -511,7 +511,6 @@ public void commitBlock(A0BlockHeader blockHeader) { try { worldState.sync(); - detailsDS.syncLargeStorage(); if (pruneEnabled) { if (stateDSPrune.isArchiveEnabled() && blockHeader.getNumber() % archiveRate == 0) { diff --git a/modAionImpl/test/org/aion/db/AionContractDetailsTest.java b/modAionImpl/test/org/aion/db/AionContractDetailsTest.java index 255c1ce827..8fd0b14931 100644 --- a/modAionImpl/test/org/aion/db/AionContractDetailsTest.java +++ b/modAionImpl/test/org/aion/db/AionContractDetailsTest.java @@ -1,5 +1,6 @@ package org.aion.db; +import static org.aion.mcf.db.DatabaseUtils.connectAndOpen; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -11,6 +12,9 @@ import org.aion.interfaces.db.ContractDetails; import org.aion.interfaces.db.PruneConfig; import org.aion.interfaces.db.RepositoryConfig; +import org.aion.log.AionLoggerFactory; +import org.aion.log.LogEnum; +import org.aion.mcf.trie.JournalPruneDataSource; import org.aion.mcf.vm.types.DataWordImpl; import org.aion.types.Address; import org.aion.types.ByteArrayWrapper; @@ -24,8 +28,11 @@ import org.aion.zero.impl.db.ContractDetailsAion; import org.apache.commons.lang3.RandomUtils; import org.junit.Test; +import org.slf4j.Logger; public class AionContractDetailsTest { + private static final Logger LOG = AionLoggerFactory.getLogger(LogEnum.DB.name()); + private static final int IN_MEMORY_STORAGE_LIMIT = 1000000; // CfgAion.inst().getDb().getDetailsInMemoryStorageLimit(); @@ -289,6 +296,72 @@ public void testExternalStorageSerialization() { deserialized.delete(new DataWordImpl(RandomUtils.nextBytes(16)).toWrapper()); } + @Test + public void testContractStorageSwitch() { + Address address = Address.wrap(RandomUtils.nextBytes(Address.SIZE)); + byte[] code = RandomUtils.nextBytes(512); + Map elements = new HashMap<>(); + + int memstoragelimit = 512; + AionContractDetailsImpl original = new AionContractDetailsImpl(0, memstoragelimit); + + // getting storage specific properties + Properties sharedProps; + sharedProps = repoConfig.getDatabaseConfig("storage"); + sharedProps.setProperty(DatabaseFactory.Props.ENABLE_LOCKING, "false"); + sharedProps.setProperty(DatabaseFactory.Props.DB_PATH, repoConfig.getDbPath()); + sharedProps.setProperty(DatabaseFactory.Props.DB_NAME, "storage"); + ByteArrayKeyValueDatabase storagedb = connectAndOpen(sharedProps, LOG); + JournalPruneDataSource jpd = new JournalPruneDataSource(storagedb); + original.setDataSource(jpd); + original.setAddress(address); + original.setCode(code); + + // the first 2 insertion use memory storage + for (int i = 0; i < 2; i++) { + DataWordImpl key = new DataWordImpl(RandomUtils.nextBytes(16)); + DataWordImpl value = new DataWordImpl(RandomUtils.nextBytes(16)); + + elements.put(key, value); + original.put(key.toWrapper(), wrapValueForPut(value)); + } + + original.decode(original.getEncoded()); + original.syncStorage(); + assertTrue(!original.externalStorage); + + // transfer to external storage since 3rd insert + DataWordImpl key3rd = new DataWordImpl(RandomUtils.nextBytes(16)); + DataWordImpl value = new DataWordImpl(RandomUtils.nextBytes(16)); + elements.put(key3rd, value); + original.put(key3rd.toWrapper(), wrapValueForPut(value)); + + original.decode(original.getEncoded()); + original.syncStorage(); + assertTrue(original.externalStorage); + + byte[] rlp = original.getEncoded(); + + AionContractDetailsImpl deserialized = new AionContractDetailsImpl(0, memstoragelimit); + deserialized.setDataSource(jpd); + deserialized.decode(rlp); + + assertTrue(deserialized.externalStorage); + assertEquals(address, deserialized.getAddress()); + assertEquals(ByteUtil.toHexString(code), ByteUtil.toHexString(deserialized.getCode())); + + for (DataWordImpl key : elements.keySet()) { + assertEquals( + elements.get(key).toWrapper(), + wrapValueFromGet(deserialized.get(key.toWrapper()))); + } + + DataWordImpl deletedKey = elements.keySet().iterator().next(); + + deserialized.delete(deletedKey.toWrapper()); + deserialized.delete(new DataWordImpl(RandomUtils.nextBytes(16)).toWrapper()); + } + @Test public void testExternalStorageTransition() { Address address = Address.wrap(RandomUtils.nextBytes(Address.SIZE)); diff --git a/modAionImpl/test/org/aion/zero/impl/BlockchainAccountStateBenchmark.java b/modAionImpl/test/org/aion/zero/impl/BlockchainAccountStateBenchmark.java index 5985c65eae..221560c14f 100644 --- a/modAionImpl/test/org/aion/zero/impl/BlockchainAccountStateBenchmark.java +++ b/modAionImpl/test/org/aion/zero/impl/BlockchainAccountStateBenchmark.java @@ -1,14 +1,19 @@ package org.aion.zero.impl; import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import java.io.File; import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.util.ArrayList; +import java.security.SecureRandom; import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Random; +import java.util.ArrayList; +import java.util.Collections; + import org.aion.types.Address; import org.aion.crypto.ECKey; import org.aion.crypto.HashUtil; @@ -17,6 +22,9 @@ import org.aion.mcf.core.ImportResult; import org.aion.util.bytes.ByteUtil; +import org.aion.util.conversions.Hex; +import org.aion.zero.db.AionContractDetailsImpl; + import org.aion.zero.impl.types.AionBlock; import org.aion.zero.impl.types.AionTxInfo; import org.aion.zero.types.AionTransaction; @@ -30,9 +38,11 @@ @RunWith(Parameterized.class) public class BlockchainAccountStateBenchmark { - public static String baseTestPath = "test_db"; + private Random random = new SecureRandom(); + + private static String baseTestPath = "test_db"; - public static String[] dbPaths = { + private static String[] dbPaths = { "level_db_state_test", "level_db_expansion_test", "h2_db_state_test", @@ -41,7 +51,7 @@ public class BlockchainAccountStateBenchmark { "rocks_db_expansion_test" }; - public static void resetFileState() { + private static void resetFileState() { File f = new File(baseTestPath); if (f.exists()) { FileUtils.deleteRecursively(f); @@ -124,12 +134,9 @@ public String getDbPath() { }); } - private String name; - private StandaloneBlockchain.Bundle bundle; - public BlockchainAccountStateBenchmark(String name, StandaloneBlockchain.Bundle bundle) { - this.name = name; + public BlockchainAccountStateBenchmark(StandaloneBlockchain.Bundle bundle) { this.bundle = bundle; } @@ -192,7 +199,7 @@ private static AionBlock createBundleAndCheck( } private static final String STATE_EXPANSION_BYTECODE = - "0x605060405260006001600050909055341561001a5760006000fd5b61001f565b6101688061002e6000396000f30060506040526000356c01000000000000000000000000900463ffffffff16806331e658a514610049578063549262ba1461008957806361bc221a1461009f57610043565b60006000fd5b34156100555760006000fd5b610073600480808060100135903590916020019091929050506100c9565b6040518082815260100191505060405180910390f35b34156100955760006000fd5b61009d6100eb565b005b34156100ab5760006000fd5b6100b3610133565b6040518082815260100191505060405180910390f35b6000600050602052818160005260105260306000209050600091509150505481565b6001600060005060006001600050546000825281601001526020019081526010016000209050600050819090905550600160008181505480929190600101919050909055505b565b600160005054815600a165627a7a72305820c615f3373321aa7e9c05d9a69e49508147861fb2a54f2945fbbaa7d851125fe80029"; + "605060405234156100105760006000fd5b610015565b610146806100246000396000f30060506040526000356c01000000000000000000000000900463ffffffff16806326121ff0146100335761002d565b60006000fd5b341561003f5760006000fd5b610047610049565b005b6000600050805480600101828161006091906100b3565b91909060005260106000209050906002020160005b7e112233445566778899001122334455667788990011223344556677889900119091929091925091909060001916909091806001018390555550505b565b8154818355818115116100e25760020281600202836000526010600020905091820191016100e191906100e7565b5b505050565b61011791906100f1565b80821115610113576000818150806000905560010160009055506002016100f1565b5090565b905600a165627a7a72305820c4bdcf87b810c9e707e3df169b98d6a37a6e6f3356cc8c120ea06c64696f85c20029"; @Ignore @Test @@ -229,7 +236,7 @@ public void testExpandOneAccountStorage() throws InterruptedException { } } - public static Pair createContract( + private static Pair createContract( StandaloneBlockchain bc, ECKey key, AionBlock parentBlock) { BigInteger accountNonce = bc.getRepository().getNonce(new Address(key.getAddress())); @@ -244,7 +251,7 @@ public static Pair createContract( 1); creationTx.sign(key); - AionBlock block = bc.createNewBlock(parentBlock, Arrays.asList(creationTx), true); + AionBlock block = bc.createNewBlock(parentBlock, Collections.singletonList(creationTx), true); return Pair.of(block, creationTx.getTransactionHash()); } @@ -253,22 +260,29 @@ private static AionBlock createContractBundle( final ECKey key, final AionBlock parentBlock, final Address contractAddress) { + return createContractBundle(bc, key, parentBlock, contractAddress, 133); + } + + private static AionBlock createContractBundle( + final StandaloneBlockchain bc, + final ECKey key, + final AionBlock parentBlock, + final Address contractAddress, + final int repeat) { + BigInteger accountNonce = bc.getRepository().getNonce(new Address(key.getAddress())); List transactions = new ArrayList<>(); - // command - ByteBuffer buf = ByteBuffer.allocate(4); - buf.put(HashUtil.keccak256("put()".getBytes()), 0, 4); + byte[] callData = Hex.decode("26121ff0"); - // create 400 transactions per bundle // byte[] nonce, Address to, byte[] value, byte[] data, long nrg, long nrgPrice - for (int i = 0; i < 133; i++) { + for (int i = 0; i < repeat; i++) { AionTransaction sendTransaction = new AionTransaction( accountNonce.toByteArray(), contractAddress, BigInteger.ZERO.toByteArray(), - buf.array(), + callData, 200000, 1); sendTransaction.sign(key); @@ -278,7 +292,7 @@ private static AionBlock createContractBundle( AionBlock block = bc.createNewBlock(parentBlock, transactions, true); - assertThat(block.getTransactionsList().size()).isEqualTo(133); + assertThat(block.getTransactionsList().size()).isEqualTo(repeat); // clear the trie bc.getRepository().flush(); @@ -291,4 +305,48 @@ private static AionBlock createContractBundle( assertThat(result).isEqualTo(ImportResult.IMPORTED_BEST); return block; } + + @Test + public void testExpandContractsStorage() throws InterruptedException { + try { + StandaloneBlockchain.Bundle bundle = this.bundle; + + StandaloneBlockchain bc = bundle.bc; + + int r = random.nextInt(bundle.privateKeys.size()); + ECKey key = bundle.privateKeys.get(r); + // deploy contract + Pair res = createContract(bc, key, bc.getGenesis()); + bc.tryToConnect(res.getLeft()); + AionTxInfo info = bc.getTransactionInfo(res.getRight()); + assertThat(info.getReceipt().isValid()).isTrue(); + + Address contractAddress = info.getReceipt().getTransaction().getContractAddress(); + + byte[] contractCode = + bc.getRepository() + .getCode(info.getReceipt().getTransaction().getContractAddress()); + + + System.out.println("deployed contract code: " + ByteUtil.toHexString(contractCode)); + System.out.println("deployed at: " + contractAddress); + + AionContractDetailsImpl acdi = new AionContractDetailsImpl(bc.getRepository().getContractDetails(contractAddress).getEncoded()); + assertFalse(acdi.externalStorage); + + // around 350 tx to letting the contract storage from memory switch to the external storage. + for (int i = 0; i < 9; i++) { + createContractBundle(bc, key, bc.getBestBlock(), contractAddress, 50); + } + + acdi = new AionContractDetailsImpl(bc.getRepository().getContractDetails(contractAddress).getEncoded()); + assertTrue(acdi.externalStorage); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + bundle.bc.getRepository().close(); + Thread.sleep(1000L); + } + } } diff --git a/modDbImpl/src/module-info.java b/modDbImpl/src/module-info.java index 6b57ada028..62b434d9ba 100644 --- a/modDbImpl/src/module-info.java +++ b/modDbImpl/src/module-info.java @@ -7,6 +7,7 @@ requires h2.mvstore; requires com.google.common; requires mongo.java.driver; + requires leveldbjni.all; exports org.aion.db.impl; exports org.aion.db.impl.leveldb; diff --git a/modMcf/build.gradle b/modMcf/build.gradle index 17d95b08d9..b9c21305c3 100644 --- a/modMcf/build.gradle +++ b/modMcf/build.gradle @@ -28,6 +28,8 @@ dependencies { testCompile group: 'commons-codec', name: 'commons-codec', version: '1.10' testCompile "org.mockito:mockito-core:2.23.0" testCompile 'pl.pragmatists:JUnitParams:1.1.1' + testCompile 'network.aion:crypto4j:0.4.0' + } // Skip unit tests when doing build task; unit tests are all mixed up with diff --git a/modMcf/src/org/aion/mcf/trie/TrieImpl.java b/modMcf/src/org/aion/mcf/trie/TrieImpl.java index ac546947b3..8e5cc9de65 100644 --- a/modMcf/src/org/aion/mcf/trie/TrieImpl.java +++ b/modMcf/src/org/aion/mcf/trie/TrieImpl.java @@ -473,7 +473,7 @@ private boolean isEmptyNode(Object node) { || n.length() == 0); } - private Object[] copyNode(Value currentNode) { + private static Object[] copyNode(Value currentNode) { Object[] itemList = emptyStringSlice(LIST_SIZE); for (int i = 0; i < LIST_SIZE; i++) { Object cpy = currentNode.get(i).asObj(); diff --git a/modRlp/src/main/java/org/aion/rlp/Value.java b/modRlp/src/main/java/org/aion/rlp/Value.java index 4adeb07ea5..8fb5a5a0d4 100644 --- a/modRlp/src/main/java/org/aion/rlp/Value.java +++ b/modRlp/src/main/java/org/aion/rlp/Value.java @@ -147,13 +147,14 @@ public byte[] getData() { public Value get(int index) { if (isList()) { // Guard for OutOfBounds - if (asList().size() <= index) { + List list = asList(); + if (list.size() <= index) { return new Value(null); } if (index < 0) { throw new RuntimeException("Negative index not allowed"); } - return new Value(asList().get(index)); + return new Value(list.get(index)); } // If this wasn't a slice you probably shouldn't be using this function return new Value(null);