diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index d7dd8f5009..6e0f16a53d 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -1,4 +1,4 @@ -name: "hugegraph-pd-store-ci" +name: "pd-store-ci" on: push: @@ -109,10 +109,39 @@ jobs: cp $HOME/.m2/settings.xml /tmp/settings.xml mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml + - name: Package + run: | + mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp + + - name: Prepare env and service + run: | + $TRAVIS_DIR/start-pd.sh + $TRAVIS_DIR/start-store.sh + - name: Run common test run: | mvn test -pl hugegraph-store/hg-store-test -am -P store-common-test + - name: Run client test + run: | + mvn test -pl hugegraph-store/hg-store-test -am -P store-client-test + + - name: Run core test + run: | + mvn test -pl hugegraph-store/hg-store-test -am -P store-core-test + + - name: Run rocksdb test + run: | + mvn test -pl hugegraph-store/hg-store-test -am -P store-rocksdb-test + + - name: Run server test + run: | + mvn test -pl hugegraph-store/hg-store-test -am -P store-server-test + + - name: Run raft-core test + run: | + mvn test -pl hugegraph-store/hg-store-test -am -P store-raftcore-test + - name: Upload coverage to Codecov uses: codecov/codecov-action@v3.0.0 with: diff --git a/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreNodeSessionImpl.java b/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreNodeSessionImpl.java index 77c8a45537..73e95515c7 100644 --- a/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreNodeSessionImpl.java +++ b/hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/GrpcStoreNodeSessionImpl.java @@ -220,7 +220,10 @@ public boolean merge(String table, HgOwnerKey key, byte[] value) { private boolean prepareBatchEntry(OpType opType, String table , HgOwnerKey startKey, HgOwnerKey endKey, byte[] value) { this.batchEntryBuilder.clear().setOpType(opType); - this.batchEntryBuilder.setTable(tables.get(table)); + Integer tableCode = tables.get(table); + if (tableCode != null) { + this.batchEntryBuilder.setTable(tableCode); + } if (startKey != null) { this.batchEntryBuilder.setStartKey(toKey(startKey)); } diff --git a/hugegraph-store/hg-store-test/pom.xml b/hugegraph-store/hg-store-test/pom.xml index 2ecadb7de7..59f87eaa6d 100644 --- a/hugegraph-store/hg-store-test/pom.xml +++ b/hugegraph-store/hg-store-test/pom.xml @@ -98,10 +98,10 @@ - - - - + + org.apache.hugegraph + hg-store-core + org.apache.hugegraph hg-pd-grpc @@ -203,12 +203,12 @@ - - - - - - + + org.apache.hugegraph + hg-store-node + ${revision} + compile + @@ -217,31 +217,30 @@ maven-surefire-plugin 2.20 - - - - - - - - - - - - - - - - - - - - - - - - - + + store-client-test + + ${basedir}/src/main/java/ + + ${basedir}/target/classes/ + + + **/ClientSuiteTest.java + + + + + store-core-test + + ${basedir}/src/main/java/ + + ${basedir}/target/classes/ + + + **/CoreSuiteTest.java + + + store-common-test @@ -254,45 +253,45 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + store-rocksdb-test + + ${basedir}/src/main/java/ + + ${basedir}/target/classes/ + + + **/RocksDbSuiteTest.java + + + + + store-server-test + + ${basedir}/src/main/java/ + + ${basedir}/target/classes/ + + + **/ServerSuiteTest.java + + + **/node/**/* + + + + + store-raftcore-test + + ${basedir}/src/main/java/ + + ${basedir}/target/classes/ + + + **/RaftSuiteTest.java + + + diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/HgBusinessImplTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/HgBusinessImplTest.java new file mode 100644 index 0000000000..15deaa8462 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/HgBusinessImplTest.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; + +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.store.business.BusinessHandler; +import org.apache.hugegraph.store.business.BusinessHandlerImpl; +import org.apache.hugegraph.store.meta.PartitionManager; +import org.apache.hugegraph.store.options.HgStoreEngineOptions; +import org.apache.hugegraph.store.options.RaftRocksdbOptions; +import org.apache.hugegraph.store.pd.FakePdServiceProvider; +import org.apache.hugegraph.store.pd.PdProvider; +import org.apache.hugegraph.util.Bytes; +import org.junit.Assert; + +public class HgBusinessImplTest { + static String GRAPH_NAME = "graph_test"; + static String[] GRAPH_TABLE = {"table1", "table2"}; + + static String dbPath = "tmp/junit"; + + // @BeforeClass + public static void init() { + UnitTestBase.deleteDir(new File(dbPath)); + Map configMap = new HashMap<>(); + configMap.put("rocksdb.write_buffer_size", "1048576"); + + RaftRocksdbOptions.initRocksdbGlobalConfig(configMap); + BusinessHandlerImpl.initRocksdb(configMap, null); + } + + public BusinessHandler getBusinessHandler() { + + int partitionCount = 2; + HgStoreEngineOptions options = new HgStoreEngineOptions() {{ + setDataPath(dbPath); + setFakePdOptions(new HgStoreEngineOptions.FakePdOptions() {{ + setPartitionCount(partitionCount); + setPeersList("127.0.0.1"); + setStoreList("127.0.0.1"); + + }}); + }}; + + PdProvider pdProvider = new FakePdServiceProvider(options.getFakePdOptions()); + PartitionManager partitionManager = new PartitionManager(pdProvider, options) { + @Override + public List getLeaderPartitionIds(String graph) { + List ids = new ArrayList<>(); + for (int i = 0; i < partitionCount; i++) { + ids.add(i); + } + return ids; + } + }; + BusinessHandler handler = new BusinessHandlerImpl(partitionManager); + + return handler; + } + + // @Test + public void testPut() { + String graph1 = "test-graph11"; + String graph2 = "test-graph12"; + String table = "test"; + BusinessHandler handler = getBusinessHandler(); + handler.doPut(graph1, 0, table, "key1".getBytes(), "value1".getBytes()); + handler.doPut(graph1, 0xF, table, "key2".getBytes(), "value1".getBytes()); + handler.doPut(graph1, 0xFF, table, "key3".getBytes(), "value1".getBytes()); + handler.doPut(graph1, 0xFFF, table, "key4".getBytes(), "value1".getBytes()); + + handler.doPut(graph2, 0, table, "key21".getBytes(), "value1".getBytes()); + handler.doPut(graph2, 0xF, table, "key22".getBytes(), "value1".getBytes()); + handler.doPut(graph2, 0xFF, table, "key23".getBytes(), "value1".getBytes()); + handler.doPut(graph2, 0xFFF, table, "key24".getBytes(), "value1".getBytes()); + + System.out.println("--------------------dump all -------"); + dump(handler, graph1, 0); + System.out.println("--------------------dump scan 0 0xff -------"); + ScanIterator iterator = handler.scan(graph1, table, 0, 0xff); + int count = 0; + while (iterator.hasNext()) { + RocksDBSession.BackendColumn entry = iterator.next(); + System.out.println(new String(entry.name) + " -- " + Bytes.toHex(entry.name)); + count++; + } + + Assert.assertEquals(2, count); + + System.out.println("--------------------dump scan prefix -------"); + iterator = handler.scanPrefix(graph1, 0, table, "key".getBytes()); + + count = 0; + while (iterator.hasNext()) { + RocksDBSession.BackendColumn entry = iterator.next(); + System.out.println(new String(entry.name) + " -- " + Bytes.toHex(entry.name)); + count++; + } + + Assert.assertEquals(4, count); + System.out.println("--------------------dump scan range key1 key4 -------"); + iterator = handler.scan(graph1, 0, table, "key1".getBytes(), "key4".getBytes(), + ScanIterator.Trait.SCAN_LT_END); + count = 0; + while (iterator.hasNext()) { + RocksDBSession.BackendColumn entry = iterator.next(); + System.out.println(new String(entry.name) + " -- " + Bytes.toHex(entry.name)); + count++; + } + + Assert.assertEquals(3, count); + } + + // @Test + public void testLoadSnapshot() throws InterruptedException { + + BusinessHandler handler = getBusinessHandler(); + String graph1 = "test-graph1"; + String graph2 = "test-graph2"; + String table = "test"; + for (int i = 0; i < 256; i++) { + handler.doPut(graph1, i, table, ("key" + i).getBytes(), "value1".getBytes()); + } + ScanIterator iterator = handler.scanAll(graph1, table); + + System.out.println(iterator.count()); + + String snapshotPath; + try (RocksDBSession session = handler.getSession(0)) { + snapshotPath = session.getDbPath(); + } + + handler.closeAll(); + + System.out.println("start loadSnapshot"); + handler.loadSnapshot(snapshotPath, graph1, 0, 10); + iterator = handler.scanAll(graph1, table); + Assert.assertEquals(255, iterator.count()); + try (RocksDBSession session = handler.getSession(0)) { + System.out.println(session.getDbPath()); + } + + CountDownLatch latch = new CountDownLatch(1); + RocksDBFactory.getInstance() + .addRocksdbChangedListener(new RocksDBFactory.RocksdbChangedListener() { + @Override + public void onCompacted(String dbName) { + RocksDBFactory.RocksdbChangedListener.super.onCompacted(dbName); + } + + @Override + public void onDBDeleteBegin(String dbName, String filePath) { + RocksDBFactory.RocksdbChangedListener.super.onDBDeleteBegin(dbName, + filePath); + } + + @Override + public void onDBDeleted(String dbName, String filePath) { + latch.countDown(); + } + + @Override + public void onDBSessionReleased(RocksDBSession dbSession) { + RocksDBFactory.RocksdbChangedListener.super.onDBSessionReleased( + dbSession); + } + }); + latch.await(); + + } + + public void dump(BusinessHandler handler, String graph, int partId) { + ScanIterator cfIterator = handler.scanRaw(graph, partId, 0); + while (cfIterator.hasNext()) { + try (ScanIterator iterator = cfIterator.next()) { + byte[] cfName = cfIterator.position(); + System.out.println(graph + "-" + +partId + "-" + new String(cfName) + "--------"); + while (iterator.hasNext()) { + RocksDBSession.BackendColumn col = iterator.next(); + System.out.println(new String(col.name) + " -- " + Bytes.toHex(col.name)); + } + } + } + cfIterator.close(); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/HgSessionManagerOneRaftFakePDTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/HgSessionManagerOneRaftFakePDTest.java new file mode 100644 index 0000000000..b4e19f104d --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/HgSessionManagerOneRaftFakePDTest.java @@ -0,0 +1,481 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store; + +import static org.apache.hugegraph.store.client.util.HgAssert.isInvalid; +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.ALL_PARTITION_OWNER; +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +import org.apache.hugegraph.pd.common.PartitionUtils; +import org.apache.hugegraph.store.client.HgStoreNodeManager; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.util.HgStoreTestUtil; +import org.junit.Assert; + +/** + * 使用fake-pd，支持raft的单元测试 + */ +public class HgSessionManagerOneRaftFakePDTest { + private static final Map leaderMap = new ConcurrentHashMap<>(); + private static final Map storeMap = new ConcurrentHashMap<>(); + + private static final int partitionCount = 3; + // 需要与store的application.yml的fake-pd.partition-count保持一致 + private static final String[] storeAddress = { + "127.0.0.1:8500" + }; + + // @BeforeClass + public static void init() { + for (String address : storeAddress) { + storeMap.put((long) address.hashCode(), address); + } + for (int i = 0; i < partitionCount; i++) { + leaderMap.put(i, storeMap.keySet().iterator().next()); + } + + HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance(); + nodeManager.setNodePartitioner((builder, graphName, startKey, endKey) -> { + int startCode = PartitionUtils.calcHashcode(startKey); + int endCode = PartitionUtils.calcHashcode(endKey); + if (ALL_PARTITION_OWNER == startKey) { + storeMap.forEach((k, v) -> { + builder.add(k, -1); + }); + } else if (endKey == HgStoreClientConst.EMPTY_BYTES || startKey == endKey || + Arrays.equals(startKey, endKey)) { + builder.add(leaderMap.get(startCode % partitionCount), startCode); + } else { + Assert.fail("OwnerKey转成HashCode后已经无序了， 按照OwnerKey范围查询没意义"); + builder.add(leaderMap.get(startCode % partitionCount), startCode); + builder.add(leaderMap.get(endCode % partitionCount), endCode); + } + return 0; + }); + nodeManager.setNodeProvider((graphName, nodeId) -> { + System.out.println("HgStoreNodeProvider apply " + graphName + " " + nodeId + " " + + storeMap.get(nodeId)); + return nodeManager.getNodeBuilder().setNodeId(nodeId) + .setAddress(storeMap.get(nodeId)).build(); + }); + nodeManager.setNodeNotifier((graphName, storeNotice) -> { + System.out.println("recv node notifier " + storeNotice); + if (storeNotice.getPartitionLeaders().size() > 0) { + leaderMap.putAll(storeNotice.getPartitionLeaders()); + System.out.println("leader changed "); + leaderMap.forEach((k, v) -> { + System.out.print(" " + k + " " + v + ","); + }); + System.out.println(); + } + return 0; + }); + } + + private static HgStoreSession getStoreSession() { + return HgSessionManager.getInstance().openSession(HgStoreTestUtil.GRAPH_NAME); + } + + private static HgStoreSession getStoreSession(String graph) { + return HgSessionManager.getInstance().openSession(graph); + } + + // @Test + public void put_get() { + System.out.println("--- test put & get ---"); + + HgStoreSession session = getStoreSession(); + + HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY"); + //HgOwnerKey key = toAllNodeKey("FOR-PUT-KEY"); + byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); + + Assert.assertTrue(session.put(HgStoreTestUtil.TABLE_NAME, key, value)); + + Assert.assertEquals(HgStoreTestUtil.toStr(value), HgStoreTestUtil.toStr(session.get( + HgStoreTestUtil.TABLE_NAME, key))); + } + + // @Test + public void put_get2() { + System.out.println("--- test put & get 2---"); + + String GRAPH_NAME = "default/hugegraph/g2"; + String TABLE_NAME = "put_get2"; + + HgStoreSession session = getStoreSession(GRAPH_NAME); + + HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY"); + //HgOwnerKey key = toAllNodeKey("FOR-PUT-KEY"); + byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); + + Assert.assertTrue(session.put(TABLE_NAME, key, value)); + + Assert.assertEquals(HgStoreTestUtil.toStr(value), + HgStoreTestUtil.toStr(session.get(TABLE_NAME, key))); + + HgKvIterator iterator = session.scanIterator(TABLE_NAME); + while (iterator.hasNext()) { + HgKvEntry entry = iterator.next(); + + System.out.println(new String(entry.key())); + Assert.assertEquals(HgStoreTestUtil.toStr(key.getKey()), + HgStoreTestUtil.toStr(entry.key())); + } + + } + + + // @Test + public void batchGet() { + System.out.println("--- test batchGet ---"); + HgStoreSession session = getStoreSession(); + String keyPrefix = "BATCH-GET-UNIT"; + + Map map = HgStoreTestUtil.batchPut(session, keyPrefix); + List keyList = + map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()); + + //printOwner(keyList); + List resList = session.batchGetOwner(HgStoreTestUtil.TABLE_NAME, keyList); + + Assert.assertFalse(isInvalid(resList)); + Assert.assertEquals(resList.size(), keyList.size()); + + // println(list); + HgStoreTestUtil.println("--- batch-get result ---"); + Assert.assertTrue((resList.stream() + .map(e -> map.containsKey(HgStoreTestUtil.toOwnerKey(e.key()))) + .allMatch(Boolean::booleanValue)) + ); + + } + + // @Test + //CAUTION: ONLY FOR LONG！ + //注意：目前只可以对long类型value进行Merge操作。 + public void merge() { + System.out.println("--- test merge (1+1=2)---"); + HgStoreSession session = getStoreSession(); + String mergeKey = "merge-key"; + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(mergeKey); + byte[] value = HgStoreTestUtil.toBytes(1L); + + System.out.println("- put " + mergeKey + ":1 -"); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); + System.out.println("- merge " + mergeKey + ":1 -"); + session.merge(HgStoreTestUtil.TABLE_NAME, key, value); + long res = HgStoreTestUtil.toLong(session.get(HgStoreTestUtil.TABLE_NAME, key)); + System.out.printf("after merge " + mergeKey + "=%s%n", res); + Assert.assertEquals(2L, res); + + String putStr = "19"; + session.put(HgStoreTestUtil.TABLE_NAME, key, HgStoreTestUtil.toBytes(putStr)); + byte[] b1 = session.get(HgStoreTestUtil.TABLE_NAME, key); + Assert.assertEquals(putStr, HgStoreTestUtil.toStr(b1)); + } + + // @Test + public void delete() { + System.out.println("--- test delete ---"); + HgStoreSession session = getStoreSession(); + + String delKey = "del-key"; + String delValue = "del-value"; + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); + byte[] value = HgStoreTestUtil.toBytes(delValue); + + HgStoreTestUtil.println("- put " + delKey + ": " + delValue); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); + HgStoreTestUtil.println("- delete " + delKey); + session.delete(HgStoreTestUtil.TABLE_NAME, key); + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println("- get " + delKey + ": " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals(EMPTY_BYTES, value); + } + + // @Test + public void deleteSingle() { + System.out.println("--- test deleteSingle ---"); + HgStoreSession session = getStoreSession(); + + String delKey = "del-single-key"; + String delValue = "del-single-value"; + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); + byte[] value = HgStoreTestUtil.toBytes(delValue); + + HgStoreTestUtil.println("- put [" + delKey + "] = " + delValue); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); + + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println( + "- before del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals(delValue, HgStoreTestUtil.toStr(value)); + + HgStoreTestUtil.println("- delete-single : [" + delKey + "]"); + session.deleteSingle(HgStoreTestUtil.TABLE_NAME, key); + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println( + "- after del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals("", HgStoreTestUtil.toStr(value)); + + } + + // @Test + public void deleteRange() { + HgStoreTestUtil.println("--- test deleteRange ---"); + HgStoreSession session = getStoreSession(); + + String rangePrefix = "DEL-RANGE-KEY"; + String owner = "batch-delete-owner"; + Map map = + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, rangePrefix, 10, + key -> { + return HgStoreTestUtil.toOwnerKey(owner, key); + }); + + HgOwnerKey startKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-00"); + HgOwnerKey endKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-05"); + + Assert.assertTrue(session.deleteRange(HgStoreTestUtil.TABLE_NAME, startKey, endKey)); + + HgStoreTestUtil.println("- after delete range from [" + + HgStoreTestUtil.toStr(startKey.getKey()) + + "] to [" + + HgStoreTestUtil.toStr(endKey.getKey()) + "]"); + + for (int i = 0; i < 10; i++) { + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-" + + HgStoreTestUtil.padLeftZeros( + String.valueOf(i), 2)); + String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key)); + HgStoreTestUtil.println( + "- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value); + + // TODO: [start,end)? + if (i < 5) { + Assert.assertEquals("", value); + } else { + //println(value); + Assert.assertNotEquals("", value); + } + } + + } + + // @Test + public void deletePrefix() { + System.out.println("--- test deletePrefix ---"); + HgStoreSession session = getStoreSession(); + + String prefixStr = "DEL-PREFIX-KEY"; + String owner = "batch-delete-owner"; + Map map = + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, prefixStr, 10, + key -> { + return HgStoreTestUtil.toOwnerKey(owner, key); + }); + + //printOwner(map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList())); + + HgOwnerKey prefix = HgStoreTestUtil.toOwnerKey(owner, prefixStr); + + Assert.assertEquals(10, HgStoreTestUtil.amountOf(session.scanIterator( + HgStoreTestUtil.TABLE_NAME, prefix))); + session.deletePrefix(HgStoreTestUtil.TABLE_NAME, prefix); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator( + HgStoreTestUtil.TABLE_NAME, prefix))); + + HgStoreTestUtil.println("- after delete by prefix:[" + prefixStr + "]"); + + for (int i = 0; i < 10; i++) { + HgOwnerKey key = + HgStoreTestUtil.toOwnerKey(owner, prefixStr + HgStoreTestUtil.toSuffix(i, 2)); + String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key)); + System.out.println("- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value); + Assert.assertEquals("", value); + } + + } + + // @Test + public void scanIterator() { + HgStoreTestUtil.println("--- test scanIterator ---"); + String tableName = "UNIT_SCAN"; + String keyName = "SCAN-ITER"; + HgStoreSession session = getStoreSession(); + HgStoreTestUtil.batchPut(session, tableName, keyName, 10000); + int count = 0; + int limit = 0; + int max = 99999; + HgKvIterator iterator = null; + + HgStoreTestUtil.println("-- test 0 element --"); + iterator = session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey("__SCAN-001"), + HgStoreTestUtil.toAllPartitionKey("__SCAN-100"), 0); + Assert.assertFalse(iterator.hasNext()); + try { + iterator.next(); + Assert.fail(); + } catch (Throwable t) { + HgStoreTestUtil.println("-- test NoSuchElementException --"); + Assert.assertTrue(t instanceof NoSuchElementException); + } + + HgStoreTestUtil.println("-- test limit 1 to 10 --"); + for (int i = 1; i <= 10; i++) { + HgStoreTestUtil.println("- limit " + i + " -"); + limit = i; + iterator = session.scanIterator(tableName, + HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), + limit); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + HgStoreTestUtil.println(entry); + } + Assert.assertEquals(limit, count); + } + + HgStoreTestUtil.println("-- test limit 1 to 10 not enough --"); + for (int i = 1; i <= 10; i++) { + HgStoreTestUtil.println("- limit " + i + " -"); + limit = i; + iterator = session.scanIterator(tableName, + HgStoreTestUtil.toAllPartitionKey(keyName + "-00001"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-00005"), + limit); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + HgStoreTestUtil.println(entry); + } + if (i <= 5) { + Assert.assertEquals(limit, count); + } else { + Assert.assertEquals(5, count); + } + + } + + HgStoreTestUtil.println("-- test limit 0 (no limit) --"); + limit = 0; + iterator = + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), limit); + + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 1000 == 0) { + HgStoreTestUtil.println(entry); + } + if (count >= max) break; + } + Assert.assertEquals(10000, count); + + HgStoreTestUtil.println("-- test scan all --"); + iterator = session.scanIterator(tableName); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 1000 == 0) { + HgStoreTestUtil.println(entry); + } + if (count >= max) break; + } + Assert.assertEquals(10000, count); + + HgStoreTestUtil.println("-- test scan prefix --"); + iterator = + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-01")); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 100 == 0) { + HgStoreTestUtil.println(entry); + } + if (count >= max) break; + } + Assert.assertEquals(1000, count); + } + + + // @Test + public void truncate() { + HgStoreTestUtil.println("--- test truncate ---"); + String graph = "graph_truncate"; + String tableName = "UNIT_TRUNCATE_1"; + String keyName = "KEY_TRUNCATE"; + + HgStoreSession session = getStoreSession(graph); + HgStoreTestUtil.batchPut(session, tableName, keyName, 100); + Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + + String tableName2 = "UNIT_TRUNCATE_2"; + HgStoreTestUtil.batchPut(session, tableName2, keyName, 100); + Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); + + + session.truncate(); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); + } + + //// @Test + public void scanIteratorHuge() { + + /*************** test no limit, with 10 millions **************/ + String tableName = "UNIT_HUGE"; + String keyName = "SCAN-HUGE"; + int amount = 10_000_000; + HgStoreSession session = getStoreSession(); + + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 10)) < + 10) { + HgStoreTestUtil.batchPut(session, tableName, keyName, amount); + } + + int count = 0; + HgKvIterator iterator = session.scanIterator(tableName); + + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % (amount / 10) == 0) { + HgStoreTestUtil.println(entry); + } + } + + Assert.assertEquals(amount, count); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/HgSessionManagerRaftFakePDTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/HgSessionManagerRaftFakePDTest.java new file mode 100644 index 0000000000..d014864870 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/HgSessionManagerRaftFakePDTest.java @@ -0,0 +1,648 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store; + +import static org.apache.hugegraph.store.client.util.HgAssert.isInvalid; +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.ALL_PARTITION_OWNER; +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +import org.apache.hugegraph.pd.common.PartitionUtils; +import org.apache.hugegraph.store.client.HgStoreNodeManager; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.util.HgStoreTestUtil; +import org.junit.Assert; + +/** + * 使用fake-pd，支持raft的单元测试 + */ +public class HgSessionManagerRaftFakePDTest { + private static final Map leaderMap = new ConcurrentHashMap<>(); + private static final Map storeMap = new ConcurrentHashMap<>(); + + private static final int partitionCount = 3; + // 需要与store的application.yml的fake-pd.partition-count保持一致 + private static final String[] storeAddress = + { // 需要与store的application.yml的fake-pd.store-list保持一致 + "127.0.0.1:8501", "127.0.0.1:8502", "127.0.0.1:8503" + }; + + /*private static String[] storeAddress = { + "127.0.0.1:9080" + };*/ + // @BeforeClass + public static void init() { + for (String address : storeAddress) { + storeMap.put((long) address.hashCode(), address); + } + for (int i = 0; i < partitionCount; i++) { + leaderMap.put(i, storeMap.keySet().iterator().next()); + } + + HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance(); + nodeManager.setNodePartitioner((builder, graphName, startKey, endKey) -> { + int startCode = PartitionUtils.calcHashcode(startKey); + int endCode = PartitionUtils.calcHashcode(endKey); + if (ALL_PARTITION_OWNER == startKey) { + storeMap.forEach((k, v) -> { + builder.add(k, -1); + }); + } else if (endKey == HgStoreClientConst.EMPTY_BYTES || startKey == endKey || + Arrays.equals(startKey, endKey)) { + builder.add(leaderMap.get(startCode % partitionCount), startCode); + } else { + Assert.fail("OwnerKey转成HashCode后已经无序了， 按照OwnerKey范围查询没意义"); + builder.add(leaderMap.get(startCode % partitionCount), startCode); + builder.add(leaderMap.get(endCode % partitionCount), endCode); + } + return 0; + }); + nodeManager.setNodeProvider((graphName, nodeId) -> { + System.out.println("HgStoreNodeProvider apply " + graphName + " " + nodeId + " " + + storeMap.get(nodeId)); + return nodeManager.getNodeBuilder().setNodeId(nodeId) + .setAddress(storeMap.get(nodeId)).build(); + }); + nodeManager.setNodeNotifier((graphName, storeNotice) -> { + System.out.println("recv node notifier " + storeNotice); + if (storeNotice.getPartitionLeaders().size() > 0) { + leaderMap.putAll(storeNotice.getPartitionLeaders()); + System.out.println("leader changed "); + leaderMap.forEach((k, v) -> { + System.out.print(" " + k + " " + v + ","); + }); + System.out.println(); + } + return 0; + }); + } + + private static HgStoreSession getStoreSession() { + return HgSessionManager.getInstance().openSession(HgStoreTestUtil.GRAPH_NAME); + } + + private static HgStoreSession getStoreSession(String graph) { + return HgSessionManager.getInstance().openSession(graph); + } + + // @Test + public void put_get() { + System.out.println("--- test put & get ---"); + + HgStoreSession session = getStoreSession(); + + HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY"); + //HgOwnerKey key = toAllNodeKey("FOR-PUT-KEY"); + byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); + + Assert.assertTrue(session.put(HgStoreTestUtil.TABLE_NAME, key, value)); + + Assert.assertEquals(HgStoreTestUtil.toStr(value), HgStoreTestUtil.toStr(session.get( + HgStoreTestUtil.TABLE_NAME, key))); + } + +// // @Test +// public void batchPutOwner() { +// Map> entries = new HashMap<>(2); +// +// int ownerNum = 10; +// int keyNum = 10; +// String keyPrefix = "BATCH-UNIT-TEST-"; +// long amount = ownerNum * keyNum; +// Map kv = new HashMap<>(keyNum, 1); +// +// for (int i = 0; i < ownerNum; i++) { +// +// for (int ii = 0; ii < keyNum; ii++) { +// HgOwnerKey ownerKey = new HgOwnerKey(toBytes("OWNER-" + i) +// , toBytes(keyPrefix + i + "-" + ii)); +// byte[] value = toBytes(keyPrefix + "VALUE-" + ownerNum + "-" + ii); +// kv.put(ownerKey, value); +// } +// +// } +// +// entries.put(TABLE_NAME, kv); +// HgStoreSession session = getStoreSession(); +// Assert.assertTrue(session.batchPutOwner(entries)); +// System.out.println("put ok"); +// Assert.assertEquals(amount, amountOf(session.scanIterator(TABLE_NAME, toAllPartitionKey +// (keyPrefix)))); +// +// } + + // // @Test +// +// public void batchPutWithoutStop() { +// final int loops = 6 * 10; +// final int interval = 1000 * 10; +// for(int j=0;j> entries = new HashMap<>(2); +// +// int ownerNum = 10; +// int keyNum = 10; +// String keyPrefix = "BATCH-UNIT-TEST-"; +// long amount = ownerNum * keyNum; +// Map kv = new HashMap<>(keyNum, 1); +// +// for (int i = 0; i < ownerNum; i++) { +// +// for (int ii = 0; ii < keyNum; ii++) { +// HgOwnerKey ownerKey = new HgOwnerKey(toBytes("OWNER-" + i * j) +// , toBytes(keyPrefix + i + "-" + ii)); +// byte[] value = toBytes(keyPrefix + "VALUE-" + ownerNum + "-" + ii*j); +// kv.put(ownerKey, value); +// } +// } +// +// entries.put(TABLE_NAME, kv); +// HgStoreSession session = getStoreSession(); +// Assert.assertTrue(session.batchPutOwner(entries)); +// System.out.println("put ok"); +// +// try { +// Thread.sleep(interval); +// } catch (InterruptedException e) { +// System.out.println(e.getMessage()); +// } +// } +// } + + + // @Test + public void batchGet() { + System.out.println("--- test batchGet ---"); + HgStoreSession session = getStoreSession(); + String keyPrefix = "BATCH-GET-UNIT"; + + Map map = HgStoreTestUtil.batchPut(session, keyPrefix); + List keyList = + map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()); + + //printOwner(keyList); + List resList = session.batchGetOwner(HgStoreTestUtil.TABLE_NAME, keyList); + + Assert.assertFalse(isInvalid(resList)); + Assert.assertEquals(resList.size(), keyList.size()); + + // println(list); + HgStoreTestUtil.println("--- batch-get result ---"); + Assert.assertTrue((resList.stream() + .map(e -> map.containsKey(HgStoreTestUtil.toOwnerKey(e.key()))) + .allMatch(Boolean::booleanValue)) + ); + + } + + // @Test + //CAUTION: ONLY FOR LONG！ + //注意：目前只可以对long类型value进行Merge操作。 + public void merge() { + System.out.println("--- test merge (1+1=2)---"); + HgStoreSession session = getStoreSession(); + String mergeKey = "merge-key"; + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(mergeKey); + byte[] value = HgStoreTestUtil.toBytes(1L); + + System.out.println("- put " + mergeKey + ":1 -"); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); + System.out.println("- merge " + mergeKey + ":1 -"); + session.merge(HgStoreTestUtil.TABLE_NAME, key, value); + long res = HgStoreTestUtil.toLong(session.get(HgStoreTestUtil.TABLE_NAME, key)); + System.out.printf("after merge " + mergeKey + "=%s%n", res); + Assert.assertEquals(2L, res); + + String putStr = "19"; + session.put(HgStoreTestUtil.TABLE_NAME, key, HgStoreTestUtil.toBytes(putStr)); + byte[] b1 = session.get(HgStoreTestUtil.TABLE_NAME, key); + Assert.assertEquals(putStr, HgStoreTestUtil.toStr(b1)); + } + + // @Test + public void delete() { + System.out.println("--- test delete ---"); + HgStoreSession session = getStoreSession(); + + String delKey = "del-key"; + String delValue = "del-value"; + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); + byte[] value = HgStoreTestUtil.toBytes(delValue); + + HgStoreTestUtil.println("- put " + delKey + ": " + delValue); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); + HgStoreTestUtil.println("- delete " + delKey); + session.delete(HgStoreTestUtil.TABLE_NAME, key); + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println("- get " + delKey + ": " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals(EMPTY_BYTES, value); + } + + // @Test + public void deleteSingle() { + System.out.println("--- test deleteSingle ---"); + HgStoreSession session = getStoreSession(); + + String delKey = "del-single-key"; + String delValue = "del-single-value"; + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); + byte[] value = HgStoreTestUtil.toBytes(delValue); + + HgStoreTestUtil.println("- put [" + delKey + "] = " + delValue); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); + + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println( + "- before del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals(delValue, HgStoreTestUtil.toStr(value)); + + HgStoreTestUtil.println("- delete-single : [" + delKey + "]"); + session.deleteSingle(HgStoreTestUtil.TABLE_NAME, key); + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println( + "- after del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals("", HgStoreTestUtil.toStr(value)); + + } + + // @Test + public void deleteRange() { + HgStoreTestUtil.println("--- test deleteRange ---"); + HgStoreSession session = getStoreSession(); + + String rangePrefix = "DEL-RANGE-KEY"; + String owner = "batch-delete-owner"; + Map map = + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, rangePrefix, 10, + key -> { + return HgStoreTestUtil.toOwnerKey(owner, key); + }); + + HgOwnerKey startKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-00"); + HgOwnerKey endKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-05"); + + Assert.assertTrue(session.deleteRange(HgStoreTestUtil.TABLE_NAME, startKey, endKey)); + + HgStoreTestUtil.println("- after delete range from [" + + HgStoreTestUtil.toStr(startKey.getKey()) + + "] to [" + + HgStoreTestUtil.toStr(endKey.getKey()) + "]"); + + for (int i = 0; i < 10; i++) { + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-" + + HgStoreTestUtil.padLeftZeros( + String.valueOf(i), 2)); + String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key)); + HgStoreTestUtil.println( + "- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value); + + // TODO: [start,end)? + if (i < 5) { + Assert.assertEquals("", value); + } else { + //println(value); + Assert.assertNotEquals("", value); + } + } + + } + + // @Test + public void deletePrefix() { + System.out.println("--- test deletePrefix ---"); + HgStoreSession session = getStoreSession(); + + String prefixStr = "DEL-PREFIX-KEY"; + String owner = "batch-delete-owner"; + Map map = + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, prefixStr, 10, + key -> { + return HgStoreTestUtil.toOwnerKey(owner, key); + }); + + //printOwner(map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList())); + + HgOwnerKey prefix = HgStoreTestUtil.toOwnerKey(owner, prefixStr); + + Assert.assertEquals(10, HgStoreTestUtil.amountOf(session.scanIterator( + HgStoreTestUtil.TABLE_NAME, prefix))); + session.deletePrefix(HgStoreTestUtil.TABLE_NAME, prefix); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator( + HgStoreTestUtil.TABLE_NAME, prefix))); + + HgStoreTestUtil.println("- after delete by prefix:[" + prefixStr + "]"); + + for (int i = 0; i < 10; i++) { + HgOwnerKey key = + HgStoreTestUtil.toOwnerKey(owner, prefixStr + HgStoreTestUtil.toSuffix(i, 2)); + String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key)); + System.out.println("- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value); + Assert.assertEquals("", value); + } + + } +// // @Test +// public void batchDeleteOwner() { +// System.out.println("--- test batchDelete ---"); +// HgStoreSession session = getStoreSession(); +// String batchPrefix = "DEL-BATCH-KEY"; +// batchPut(session, batchPrefix, 10); +// +// for (int i = 0; i < 10; i++) { +// HgOwnerKey key = toOwnerKey(batchPrefix + toSuffix(i, 2)); +// String value = toStr(session.get(TABLE_NAME, key)); +// println("- get [" + toStr(key.getKey()) + "] = " + value); +// Assert.assertNotEquals("", value); +// } +// +// Set keys = new HashSet<>(); +// for (int i = 0; i < 5; i++) { +// keys.add(toOwnerKey(batchPrefix + toSuffix(i, 2))); +// } +// +// Map> map = new HashMap<>(1); +// map.put(TABLE_NAME, keys); +// session.batchDeleteOwner(map); +// +// for (int i = 0; i < 10; i++) { +// HgOwnerKey key = toOwnerKey(batchPrefix + toSuffix(i, 2)); +// String value = toStr(session.get(TABLE_NAME, key)); +// println("- get [" + toStr(key.getKey()) + "] = " + value); +// // TODO: [,)? +// if (i < 5) { +// Assert.assertEquals("", value); +// } else { +// Assert.assertNotEquals("", value); +// } +// +// } +// } + +// // @Test +// public void batchDeleteRangeOwner() { +// System.out.println("--- test batchDeleteRange ---"); +// HgStoreSession session = getStoreSession(); +// String rangePrefix = "DEL-RANGE-KEY"; +// String owner="batch-delete-owner"; +// +// batchPut(session, TABLE_NAME,rangePrefix, 10,key->{ +// return toOwnerKey(owner,key); +// }); +// batchPut(session, TABLE_NAME2, rangePrefix, 10,key->{ +// return toOwnerKey(owner,key); +// }); +// +// HgOwnerKey startKey1 = toOwnerKey(owner,rangePrefix + "-03"); +// HgOwnerKey endKey1 = toOwnerKey(owner,rangePrefix + "-05"); +// +// HgOwnerKey startKey2 = toOwnerKey(owner,rangePrefix + "-06"); +// HgOwnerKey endKey2 = toOwnerKey(owner,rangePrefix + "-09"); +// +// Map> map = new HashMap<>(); +// map.put(TABLE_NAME, new HgPair<>(startKey1, endKey1)); +// map.put(TABLE_NAME2, new HgPair<>(startKey2, endKey2)); +// +// session.batchDeleteRangeOwner(map); +// +// for (int i = 0; i < 10; i++) { +// HgOwnerKey key = toOwnerKey(owner,rangePrefix + toSuffix(i, 2)); +// String value = toStr(session.get(TABLE_NAME, key)); +// println("- get [" + toStr(key.getKey()) + "] = " + value); +// +// if (i >= 3 && i < 5) { +// Assert.assertEquals("", value); +// } else { +// Assert.assertNotEquals("", value); +// } +// +// } +// +// for (int i = 0; i < 10; i++) { +// HgOwnerKey key = toOwnerKey(owner,rangePrefix + toSuffix(i, 2)); +// String value = toStr(session.get(TABLE_NAME2, key)); +// println("- get [" + toStr(key.getKey()) + "] = " + value); +// if (i >= 6 && i < 9) { +// Assert.assertEquals("", value); +// } else { +// Assert.assertNotEquals("", value); +// } +// +// } +// } + +// // @Test +// public void batchDeletePrefix() { +// System.out.println("--- test batchDeletePrefix ---"); +// HgStoreSession session = getStoreSession(); +// String batchKey1 = "DEL-PREFIX-BATCH-1-KEY"; +// String batchKey2 = "DEL-PREFIX-BATCH-2-KEY"; +// String owner="batch-delete-owner"; +// +// batchPut(session, TABLE_NAME, batchKey1, 10,key->{ +// return toOwnerKey(owner,key); +// }); +// batchPut(session, TABLE_NAME2, batchKey2, 10,key->{ +// return toOwnerKey(owner,key); +// }); +// +// HgOwnerKey prefix1 = toOwnerKey(owner,batchKey1); +// HgOwnerKey prefix2 = toOwnerKey(owner,batchKey2); +// +// Set set = new HashSet<>(); +// +// set.add(prefix1); +// set.add(prefix2); +// +// Map> map = new HashMap<>(); +// map.put(TABLE_NAME, set); +// map.put(TABLE_NAME2, set); +// +// Assert.assertEquals(10, amountOf(session.scanIterator(TABLE_NAME, prefix1))); +// Assert.assertEquals(10, amountOf(session.scanIterator(TABLE_NAME2, prefix2))); +// +// session.batchDeletePrefixOwner(map); +// +// Assert.assertEquals(0, amountOf(session.scanIterator(TABLE_NAME, prefix1))); +// Assert.assertEquals(0, amountOf(session.scanIterator(TABLE_NAME2, prefix2))); +// +// } + + // @Test + public void scanIterator() { + HgStoreTestUtil.println("--- test scanIterator ---"); + String tableName = "UNIT_SCAN"; + String keyName = "SCAN-ITER"; + HgStoreSession session = getStoreSession(); + HgStoreTestUtil.batchPut(session, tableName, keyName, 10000); + int count = 0; + int limit = 0; + int max = 99999; + HgKvIterator iterator = null; + + HgStoreTestUtil.println("-- test 0 element --"); + iterator = session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey("__SCAN-001"), + HgStoreTestUtil.toAllPartitionKey("__SCAN-100"), 0); + Assert.assertFalse(iterator.hasNext()); + try { + iterator.next(); + Assert.fail(); + } catch (Throwable t) { + HgStoreTestUtil.println("-- test NoSuchElementException --"); + Assert.assertTrue(t instanceof NoSuchElementException); + } + + HgStoreTestUtil.println("-- test limit 1 to 10 --"); + for (int i = 1; i <= 10; i++) { + HgStoreTestUtil.println("- limit " + i + " -"); + limit = i; + iterator = session.scanIterator(tableName, + HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), + limit); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + HgStoreTestUtil.println(entry); + } + Assert.assertEquals(limit, count); + } + + HgStoreTestUtil.println("-- test limit 1 to 10 not enough --"); + for (int i = 1; i <= 10; i++) { + HgStoreTestUtil.println("- limit " + i + " -"); + limit = i; + iterator = session.scanIterator(tableName, + HgStoreTestUtil.toAllPartitionKey(keyName + "-00001"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-00005"), + limit); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + HgStoreTestUtil.println(entry); + } + if (i <= 5) { + Assert.assertEquals(limit, count); + } else { + Assert.assertEquals(5, count); + } + + } + + HgStoreTestUtil.println("-- test limit 0 (no limit) --"); + limit = 0; + iterator = + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), limit); + + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 1000 == 0) { + HgStoreTestUtil.println(entry); + } + if (count >= max) break; + } + Assert.assertEquals(10000, count); + + HgStoreTestUtil.println("-- test scan all --"); + iterator = session.scanIterator(tableName); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 1000 == 0) { + HgStoreTestUtil.println(entry); + } + if (count >= max) break; + } + Assert.assertEquals(10000, count); + + HgStoreTestUtil.println("-- test scan prefix --"); + iterator = + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-01")); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 100 == 0) { + HgStoreTestUtil.println(entry); + } + if (count >= max) break; + } + Assert.assertEquals(1000, count); + } + + + // @Test + public void truncate() { + HgStoreTestUtil.println("--- test truncate ---"); + String graph = "graph_truncate"; + String tableName = "UNIT_TRUNCATE_1"; + String keyName = "KEY_TRUNCATE"; + + HgStoreSession session = getStoreSession(graph); + HgStoreTestUtil.batchPut(session, tableName, keyName, 100); + Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + + String tableName2 = "UNIT_TRUNCATE_2"; + HgStoreTestUtil.batchPut(session, tableName2, keyName, 100); + Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); + + + session.truncate(); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); + } + + //// @Test + public void scanIteratorHuge() { + + /*************** test no limit, with 10 millions **************/ + String tableName = "UNIT_HUGE"; + String keyName = "SCAN-HUGE"; + int amount = 10_000_000; + HgStoreSession session = getStoreSession(); + + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 10)) < + 10) { + HgStoreTestUtil.batchPut(session, tableName, keyName, amount); + } + + int count = 0; + HgKvIterator iterator = session.scanIterator(tableName); + + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % (amount / 10) == 0) { + HgStoreTestUtil.println(entry); + } + } + + Assert.assertEquals(amount, count); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/HgSessionManagerRaftPDTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/HgSessionManagerRaftPDTest.java new file mode 100644 index 0000000000..9820457d78 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/HgSessionManagerRaftPDTest.java @@ -0,0 +1,981 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store; + +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.stream.Collectors; + +import org.apache.commons.io.FileUtils; +import org.apache.hugegraph.pd.client.PDClient; +import org.apache.hugegraph.pd.client.PDConfig; +import org.apache.hugegraph.store.client.grpc.KvCloseableIterator; +import org.apache.hugegraph.store.client.util.HgStoreClientConfig; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.client.util.MetricX; +import org.apache.hugegraph.store.grpc.common.ScanOrderType; +import org.apache.hugegraph.store.util.HgStoreTestUtil; +import org.junit.Assert; +import org.junit.Test; + +import lombok.extern.slf4j.Slf4j; + + +/** + * 使用pd，支持raft的单元测试 + */ +@Slf4j +public class HgSessionManagerRaftPDTest { + public final static String GRAPH_NAME_X = "default/hugegraph/x"; + public final static String GRAPH_NAME_Y = "default/hugegraph/y"; + public final static String GRAPH_NAME_Z = "default/hugegraph/z"; + public final static String TABLE_NAME_1 = "table1"; + public final static String TABLE_NAME_2 = "table2"; + public final static String TABLE_NAME_3 = "table3"; + private static final String pdAddress = "localhost:8686"; + public static HgStoreClient storeClient; + private static PDClient pdClient; + + // @BeforeClass + public static void init() { + pdClient = PDClient.create(PDConfig.of(pdAddress).setEnableCache(true)); + storeClient = HgStoreClient.create(pdClient); + } + + private static HgStoreSession getStoreSession() { + return storeClient.openSession(HgStoreTestUtil.GRAPH_NAME); + } + + private static HgStoreSession getStoreSession(String graph) { + return storeClient.openSession(graph); + } + + // @Test + public void put_get() { + System.out.println("--- test put & get ---"); + + HgStoreSession session = getStoreSession(HgStoreTestUtil.GRAPH_NAME); + + HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY"); + //HgOwnerKey key = toAllNodeKey("FOR-PUT-KEY"); + byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); + + // Assert.assertTrue(session.put(TABLE_NAME, key, value)); + + // Assert.assertEquals(toStr(value), toStr(session.get(TABLE_NAME, key))); + + long stime = System.currentTimeMillis(); + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, "BATCH-PUT-TEST", 30000); + System.out.println("Time is " + (System.currentTimeMillis() - stime)); + } + + public void put_get2() { + System.out.println("--- test put & get ---"); + + HgStoreSession session = getStoreSession("testGraph"); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY"); + byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); + long stime = System.currentTimeMillis(); + HgStoreTestUtil.batchPut(session, "testTable", "BATCH-PUT-TEST", 30000); + System.out.println("Time is " + (System.currentTimeMillis() - stime)); + } + + // @Test + public void scanPartition() { + HgStoreTestUtil.println("--- test scanPartition ---"); + + HgStoreSession session = getStoreSession(); + // System.out.println(amountOf(session.scanIterator(TABLE_NAME))); + + + Iterator iterator = session.scanIterator(HgStoreTestUtil.TABLE_NAME, 0, 65535, + HgKvStore.SCAN_HASHCODE, EMPTY_BYTES); + System.out.println(HgStoreTestUtil.amountOf(iterator)); + } + + // @Test + public void check() { + System.out.println("--- test check ---"); + + HgStoreSession session = getStoreSession(); + HgKvIterator iterator = session.scanIterator(HgStoreTestUtil.TABLE_NAME); + int count = 0; + while (iterator.hasNext()) { + iterator.next(); + count++; + } + System.out.println(count); + } + + // @Test + public void put_get_unique() { + System.out.println("--- test put & get ---"); + + HgStoreSession session = getStoreSession(); + + // add timestamp into key to avoid key duplication + Date date = new Date(); + SimpleDateFormat formatter = new SimpleDateFormat("HHmmss"); + String timestamp = formatter.format(date); + + HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY-" + timestamp); + //HgOwnerKey key = toAllNodeKey("FOR-PUT-KEY"); + byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); + + Assert.assertTrue(session.put(HgStoreTestUtil.TABLE_NAME, key, value)); + + Assert.assertEquals(HgStoreTestUtil.toStr(value), HgStoreTestUtil.toStr(session.get( + HgStoreTestUtil.TABLE_NAME, key))); + } + + + @Test + public void testBatchPutExt() throws IOException { + System.out.println("--- test batchPut ---"); + HgStoreSession session = getStoreSession(); + String keyPrefix = "BATCH-GET-UNIT"; + + Map map = HgStoreTestUtil.batchPut(session, keyPrefix, 1000); + + File outputFile = new File("tmp/batch_put_list"); + if (outputFile.exists()) { + FileUtils.forceDelete(outputFile); + } + FileUtils.forceMkdir(new File("tmp/")); + + ObjectOutputStream oo = new ObjectOutputStream(new FileOutputStream(outputFile)); + oo.writeObject(map); + oo.close(); + System.out.printf("%d entries have be put into graph %s\n", map.size(), + HgStoreTestUtil.GRAPH_NAME); + + + int count = 0; + HgKvIterator iterator = null; + iterator = session.scanIterator(HgStoreTestUtil.TABLE_NAME); + while (iterator.hasNext()) { + iterator.next(); + count++; + } + Assert.assertEquals(1000, count); + + } + + + // @Test + public void testBatchGetExt() throws IOException, ClassNotFoundException { + File outputFile = new File("tmp/batch_put_list"); + ObjectInputStream ois = new ObjectInputStream(new FileInputStream(outputFile)); + Map map = (Map) ois.readObject(); + ois.close(); + System.out.printf("%d entries get from %s\n", map.size(), outputFile.getPath()); + + HgStoreSession session = getStoreSession(); + List keyList = + map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()); + List resList = session.batchGetOwner(HgStoreTestUtil.TABLE_NAME, keyList); + + Assert.assertTrue((resList.stream() + .map(e -> map.containsKey(HgStoreTestUtil.toOwnerKey(e.key()))) + .allMatch(Boolean::booleanValue)) + ); + } + + @Test + public void testBatchPutUniqueExt() throws IOException { + System.out.println("--- test batchPut ---"); + HgStoreSession session = getStoreSession(); + + // add timestamp into key to avoid key duplication + Date date = new Date(); + SimpleDateFormat formatter = new SimpleDateFormat("HHmmss"); + String timestamp = formatter.format(date); + + String keyPrefix = "BATCH-GET-UNIT-" + timestamp; + + Map map = HgStoreTestUtil.batchPut(session, keyPrefix, 300000); + + File outputFile = new File("tmp/batch_put_list"); + if (outputFile.exists()) { + FileUtils.forceDelete(outputFile); + } + FileUtils.forceMkdir(new File("tmp/")); + + ObjectOutputStream oo = new ObjectOutputStream(new FileOutputStream(outputFile)); + oo.writeObject(map); + oo.close(); + System.out.printf("%d entries have be put into graph %s\n", map.size(), + HgStoreTestUtil.GRAPH_NAME); + } + + // @Test + public void testBatchPutMultiGraph() throws IOException { + System.out.println("--- test testBatchPutMultiGraph ---"); + Date date = new Date(); + SimpleDateFormat formatter = new SimpleDateFormat("HHmmss"); + String timestamp = formatter.format(date); + HgStoreSession session1 = getStoreSession(GRAPH_NAME_X); + HgStoreSession session2 = getStoreSession(GRAPH_NAME_Y); + HgStoreSession session3 = getStoreSession(GRAPH_NAME_Z); + String keyPrefix1 = "BATCH-PUT-UNIT-X-" + timestamp; + String keyPrefix2 = "BATCH-PUT-UNIT-Y-" + timestamp; + String keyPrefix3 = "BATCH-PUT-UNIT-Z-" + timestamp; + HgStoreTestUtil.batchPut(session1, TABLE_NAME_1, keyPrefix1, 1000); + HgStoreTestUtil.batchPut(session1, TABLE_NAME_2, keyPrefix1, 1000); + HgStoreTestUtil.batchPut(session1, TABLE_NAME_3, keyPrefix1, 1000); + HgStoreTestUtil.batchPut(session2, TABLE_NAME_1, keyPrefix2, 1000); + HgStoreTestUtil.batchPut(session2, TABLE_NAME_2, keyPrefix2, 1000); + HgStoreTestUtil.batchPut(session2, TABLE_NAME_3, keyPrefix2, 1000); + HgStoreTestUtil.batchPut(session3, TABLE_NAME_1, keyPrefix3, 1000); + HgStoreTestUtil.batchPut(session3, TABLE_NAME_2, keyPrefix3, 1000); + HgStoreTestUtil.batchPut(session3, TABLE_NAME_3, keyPrefix3, 1000); + } + + // @Test + //CAUTION: ONLY FOR LONG！ + //注意：目前只可以对long类型value进行Merge操作。 + public void merge() { + System.out.println("--- test merge (1+1=2)---"); + HgStoreSession session = getStoreSession(); + String mergeKey = "merge-key"; + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(mergeKey); + byte[] value = HgStoreTestUtil.toBytes(1L); + + System.out.println("- put " + mergeKey + ":1 -"); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); + System.out.println("- merge " + mergeKey + ":1 -"); + session.merge(HgStoreTestUtil.TABLE_NAME, key, value); + long res = HgStoreTestUtil.toLong(session.get(HgStoreTestUtil.TABLE_NAME, key)); + System.out.printf("after merge " + mergeKey + "=%s%n", res); + Assert.assertEquals(2L, res); + + String putStr = "19"; + session.put(HgStoreTestUtil.TABLE_NAME, key, HgStoreTestUtil.toBytes(putStr)); + byte[] b1 = session.get(HgStoreTestUtil.TABLE_NAME, key); + Assert.assertEquals(putStr, HgStoreTestUtil.toStr(b1)); + } + + // @Test + public void delete() { + System.out.println("--- test delete ---"); + HgStoreSession session = getStoreSession(); + + String delKey = "del-key"; + String delValue = "del-value"; + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); + byte[] value = HgStoreTestUtil.toBytes(delValue); + + HgStoreTestUtil.println("- put " + delKey + ": " + delValue); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); + HgStoreTestUtil.println("- delete " + delKey); + session.delete(HgStoreTestUtil.TABLE_NAME, key); + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println("- get " + delKey + ": " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals(EMPTY_BYTES, value); + } + + // @Test + public void deleteSingle() { + System.out.println("--- test deleteSingle ---"); + HgStoreSession session = getStoreSession(); + + String delKey = "del-single-key"; + String delValue = "del-single-value"; + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); + byte[] value = HgStoreTestUtil.toBytes(delValue); + + HgStoreTestUtil.println("- put [" + delKey + "] = " + delValue); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); + + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println( + "- before del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals(delValue, HgStoreTestUtil.toStr(value)); + + HgStoreTestUtil.println("- delete-single : [" + delKey + "]"); + session.deleteSingle(HgStoreTestUtil.TABLE_NAME, key); + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println( + "- after del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals("", HgStoreTestUtil.toStr(value)); + + } + + // @Test + public void deleteRange() { + HgStoreTestUtil.println("--- test deleteRange ---"); + HgStoreSession session = getStoreSession(); + + String rangePrefix = "DEL-RANGE-KEY"; + String owner = "batch-delete-owner"; + Map map = + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, rangePrefix, 10, + key -> { + return HgStoreTestUtil.toOwnerKey(owner, key); + }); + + HgOwnerKey startKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-00"); + HgOwnerKey endKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-05"); + + Assert.assertTrue(session.deleteRange(HgStoreTestUtil.TABLE_NAME, startKey, endKey)); + + HgStoreTestUtil.println("- after delete range from [" + + HgStoreTestUtil.toStr(startKey.getKey()) + + "] to [" + + HgStoreTestUtil.toStr(endKey.getKey()) + "]"); + + for (int i = 0; i < 10; i++) { + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-" + + HgStoreTestUtil.padLeftZeros( + String.valueOf(i), 2)); + String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key)); + HgStoreTestUtil.println( + "- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value); + + // TODO: [start,end)? + if (i < 5) { + Assert.assertEquals("", value); + } else { + //println(value); + Assert.assertNotEquals("", value); + } + } + + } + + // @Test + public void deletePrefix() { + System.out.println("--- test deletePrefix ---"); + HgStoreSession session = getStoreSession(); + + String prefixStr = "DEL-PREFIX-KEY"; + String owner = "batch-delete-owner"; + Map map = + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, prefixStr, 10, + key -> { + return HgStoreTestUtil.toOwnerKey(owner, key); + }); + + //printOwner(map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList())); + + HgOwnerKey prefix = HgStoreTestUtil.toOwnerKey(owner, prefixStr); + + Assert.assertEquals(10, HgStoreTestUtil.amountOf(session.scanIterator( + HgStoreTestUtil.TABLE_NAME, prefix))); + session.deletePrefix(HgStoreTestUtil.TABLE_NAME, prefix); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator( + HgStoreTestUtil.TABLE_NAME, prefix))); + + HgStoreTestUtil.println("- after delete by prefix:[" + prefixStr + "]"); + + for (int i = 0; i < 10; i++) { + HgOwnerKey key = + HgStoreTestUtil.toOwnerKey(owner, prefixStr + HgStoreTestUtil.toSuffix(i, 2)); + String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key)); + System.out.println("- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value); + Assert.assertEquals("", value); + } + + } +// // @Test +// public void batchDeleteOwner() { +// System.out.println("--- test batchDelete ---"); +// HgStoreSession session = getStoreSession(); +// String batchPrefix = "DEL-BATCH-KEY"; +// batchPut(session, batchPrefix, 10); +// +// for (int i = 0; i < 10; i++) { +// HgOwnerKey key = toOwnerKey(batchPrefix + toSuffix(i, 2)); +// String value = toStr(session.get(TABLE_NAME, key)); +// println("- get [" + toStr(key.getKey()) + "] = " + value); +// Assert.assertNotEquals("", value); +// } +// +// Set keys = new HashSet<>(); +// for (int i = 0; i < 5; i++) { +// keys.add(toOwnerKey(batchPrefix + toSuffix(i, 2))); +// } +// +// Map> map = new HashMap<>(1); +// map.put(TABLE_NAME, keys); +// session.batchDeleteOwner(map); +// +// for (int i = 0; i < 10; i++) { +// HgOwnerKey key = toOwnerKey(batchPrefix + toSuffix(i, 2)); +// String value = toStr(session.get(TABLE_NAME, key)); +// println("- get [" + toStr(key.getKey()) + "] = " + value); +// // TODO: [,)? +// if (i < 5) { +// Assert.assertEquals("", value); +// } else { +// Assert.assertNotEquals("", value); +// } +// +// } +// } + +// // @Test +// public void batchDeleteRangeOwner() { +// System.out.println("--- test batchDeleteRange ---"); +// HgStoreSession session = getStoreSession(); +// String rangePrefix = "DEL-RANGE-KEY"; +// String owner="batch-delete-owner"; +// +// batchPut(session, TABLE_NAME,rangePrefix, 10,key->{ +// return toOwnerKey(owner,key); +// }); +// batchPut(session, TABLE_NAME2, rangePrefix, 10,key->{ +// return toOwnerKey(owner,key); +// }); +// +// HgOwnerKey startKey1 = toOwnerKey(owner,rangePrefix + "-03"); +// HgOwnerKey endKey1 = toOwnerKey(owner,rangePrefix + "-05"); +// +// HgOwnerKey startKey2 = toOwnerKey(owner,rangePrefix + "-06"); +// HgOwnerKey endKey2 = toOwnerKey(owner,rangePrefix + "-09"); +// +// Map> map = new HashMap<>(); +// map.put(TABLE_NAME, new HgPair<>(startKey1, endKey1)); +// map.put(TABLE_NAME2, new HgPair<>(startKey2, endKey2)); +// +// session.batchDeleteRangeOwner(map); +// +// for (int i = 0; i < 10; i++) { +// HgOwnerKey key = toOwnerKey(owner,rangePrefix + toSuffix(i, 2)); +// String value = toStr(session.get(TABLE_NAME, key)); +// println("- get [" + toStr(key.getKey()) + "] = " + value); +// +// if (i >= 3 && i < 5) { +// Assert.assertEquals("", value); +// } else { +// Assert.assertNotEquals("", value); +// } +// +// } +// +// for (int i = 0; i < 10; i++) { +// HgOwnerKey key = toOwnerKey(owner,rangePrefix + toSuffix(i, 2)); +// String value = toStr(session.get(TABLE_NAME2, key)); +// println("- get [" + toStr(key.getKey()) + "] = " + value); +// if (i >= 6 && i < 9) { +// Assert.assertEquals("", value); +// } else { +// Assert.assertNotEquals("", value); +// } +// +// } +// } + +// // @Test +// public void batchDeletePrefix() { +// System.out.println("--- test batchDeletePrefix ---"); +// HgStoreSession session = getStoreSession(); +// String batchKey1 = "DEL-PREFIX-BATCH-1-KEY"; +// String batchKey2 = "DEL-PREFIX-BATCH-2-KEY"; +// String owner="batch-delete-owner"; +// +// batchPut(session, TABLE_NAME, batchKey1, 10,key->{ +// return toOwnerKey(owner,key); +// }); +// batchPut(session, TABLE_NAME2, batchKey2, 10,key->{ +// return toOwnerKey(owner,key); +// }); +// +// HgOwnerKey prefix1 = toOwnerKey(owner,batchKey1); +// HgOwnerKey prefix2 = toOwnerKey(owner,batchKey2); +// +// Set set = new HashSet<>(); +// +// set.add(prefix1); +// set.add(prefix2); +// +// Map> map = new HashMap<>(); +// map.put(TABLE_NAME, set); +// map.put(TABLE_NAME2, set); +// +// Assert.assertEquals(10, amountOf(session.scanIterator(TABLE_NAME, prefix1))); +// Assert.assertEquals(10, amountOf(session.scanIterator(TABLE_NAME2, prefix2))); +// +// session.batchDeletePrefixOwner(map); +// +// Assert.assertEquals(0, amountOf(session.scanIterator(TABLE_NAME, prefix1))); +// Assert.assertEquals(0, amountOf(session.scanIterator(TABLE_NAME2, prefix2))); +// +// } + + // @Test + public void scanIterator() { + HgStoreTestUtil.println("--- test scanIterator ---"); + String tableName = "UNIT_SCAN"; + String keyName = "SCAN-ITER"; + HgStoreSession session = getStoreSession(); + HgStoreTestUtil.batchPut(session, tableName, keyName, 10000); + int count = 0; + int limit = 0; + int max = 99999; + HgKvIterator iterator = null; + + HgStoreTestUtil.println("-- test 0 element --"); + iterator = session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey("__SCAN-001"), + HgStoreTestUtil.toAllPartitionKey("__SCAN-100"), 0); + Assert.assertFalse(iterator.hasNext()); + try { + iterator.next(); + Assert.fail(); + } catch (Throwable t) { + HgStoreTestUtil.println("-- test NoSuchElementException --"); + Assert.assertTrue(t instanceof NoSuchElementException); + } + + HgStoreTestUtil.println("-- test limit 1 to 10 --"); + for (int i = 1; i <= 10; i++) { + HgStoreTestUtil.println("- limit " + i + " -"); + limit = i; + iterator = session.scanIterator(tableName, + HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), + limit); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + HgStoreTestUtil.println(entry); + } + Assert.assertEquals(limit, count); + } + + HgStoreTestUtil.println("-- test limit 1 to 10 not enough --"); + for (int i = 1; i <= 10; i++) { + HgStoreTestUtil.println("- limit " + i + " -"); + limit = i; + iterator = session.scanIterator(tableName, + HgStoreTestUtil.toAllPartitionKey(keyName + "-00001"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-00005"), + limit); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + HgStoreTestUtil.println(entry); + } + if (i <= 5) { + Assert.assertEquals(limit, count); + } else { + Assert.assertEquals(5, count); + } + + } + + HgStoreTestUtil.println("-- test limit 0 (no limit) --"); + limit = 0; + iterator = + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), limit); + + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 1000 == 0) { + HgStoreTestUtil.println(entry); + } + if (count >= max) break; + } + Assert.assertEquals(10000, count); + + HgStoreTestUtil.println("-- test scan all --"); + iterator = session.scanIterator(tableName); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 1000 == 0) { + HgStoreTestUtil.println(entry); + } + if (count >= max) break; + } + Assert.assertEquals(10000, count); + + HgStoreTestUtil.println("-- test scan prefix --"); + iterator = + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-01")); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 100 == 0) { + HgStoreTestUtil.println(entry); + } + if (count >= max) break; + } + Assert.assertEquals(1000, count); + } + + // @Test + public void paging() { + HgStoreTestUtil.println("--- test scanIterator_range ---"); + String graph = "UNIT/paging"; + String tableName = "UNIT_SCAN_PAGING"; + String keyName = "SCAN-PAGING"; + int keyAmt = 100; + HgStoreSession session = getStoreSession(graph); + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); + int count = 0; + int limit = 0; + int max = 99999; + HgKvIterator iterator = null; + + count = 0; + + iterator = session.scanIterator(tableName + , HgStoreTestUtil.toAllPartitionKey(keyName + "-000") + , HgStoreClientConst.EMPTY_OWNER_KEY + , 0, HgKvStore.SCAN_ANY, EMPTY_BYTES + ); + //HgStoreTestUtil.println(iterator); + // Assert.assertEquals(100, HgStoreTestUtil.amountOf(iterator)); + List positionList = new LinkedList<>(); + while (iterator.hasNext()) { + HgStoreTestUtil.print((count++) + " "); + HgKvEntry entry = iterator.next(); + HgStoreTestUtil.print(entry); + HgStoreTestUtil.print(" " + Arrays.toString(iterator.position()) + "\n"); + positionList.add(iterator.position()); + if (count >= max) break; + } + + +/* iterator = session.scanIterator(tableName + , toAllPartitionKey(keyName + "-000") + , HgStoreClientConst.EMPTY_OWNER_KEY + , 0, HgKvStore.SCAN_ANY, EMPTY_BYTES + ); + + byte[] position=positionList.get(50); + println("seek: "+Arrays.toString(position)); + iterator.seek(position); + //HgStoreTestUtil.println(iterator); + Assert.assertEquals(50, HgStoreTestUtil.amountOf(iterator));*/ + + iterator = session.scanIterator(tableName, 100); + + byte[] position = positionList.get(50); + HgStoreTestUtil.println("seek: " + Arrays.toString(position)); + iterator.seek(position); + HgStoreTestUtil.println(iterator); + //Assert.assertEquals(50, HgStoreTestUtil.amountOf(iterator)); + + } + + // @Test + public void truncate() { + HgStoreTestUtil.println("--- test truncate ---"); + String graph = "graph_truncate"; + String tableName = "UNIT_TRUNCATE_1"; + String keyName = "KEY_TRUNCATE"; + + HgStoreSession session = getStoreSession(graph); + HgStoreTestUtil.batchPut(session, tableName, keyName, 100); + Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + + String tableName2 = "UNIT_TRUNCATE_2"; + HgStoreTestUtil.batchPut(session, tableName2, keyName, 100); + Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); + + + session.truncate(); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); + } + + //// @Test + public void scanIteratorHuge() { + + /*************** test no limit, with 10 millions **************/ + String tableName = "UNIT_HUGE"; + String keyName = "SCAN-HUGE"; + int amount = 10_000_000; + HgStoreSession session = getStoreSession(); + + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 10)) < + 10) { + HgStoreTestUtil.batchPut(session, tableName, keyName, amount); + } + + int count = 0; + HgKvIterator iterator = session.scanIterator(tableName); + + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % (amount / 10) == 0) { + HgStoreTestUtil.println(entry); + } + } + + Assert.assertEquals(amount, count); + } + + + // @Test + public void scanTable() { + HgStoreSession session = getStoreSession("DEFAULT/hg1/g"); + HgStoreTestUtil.println(session.scanIterator("g+v", 10)); + + + } + + // @Test + public void testDelGraph() { + HgStoreSession session = getStoreSession(); + session.deleteGraph(HgStoreTestUtil.GRAPH_NAME); + } + + // @Test + public void benchmark_scanBatch() { + HgStoreTestUtil.println("--- Benchmark scanBatch ---"); + String tableName = "Benchmark_SCAN_BATCH"; + String keyName = "SCAN-BATCH"; + int keyAmt = 30001; + + HgStoreSession session = getStoreSession(); + + /*if (amountOf(session.scanIterator(tableName, toAllPartitionKey(keyName), 1)) < 1) { + batchPut(session, tableName, keyName, keyAmt); + }*/ + HgStoreTestUtil.println("-- Starting scan --"); + MetricX metrics = MetricX.ofStart(); + //HgStoreTestUtil.println(session.scanIterator(tableName)); + List> iterators = session.scanBatch(HgScanQuery.tableOf(tableName)); + Assert.assertEquals(keyAmt, HgStoreTestUtil.amountIn(iterators)); + metrics.end(); + log.info("*************************************************"); + log.info("*********** Batch Scan Completed **************"); + log.info("Total : {} (sec)", metrics.past() / 1000); + log.info(" Sum : {} (sec)", MetricX.getIteratorWait() / 1000); + log.info(" Amt : {} (scans).", MetricX.getIteratorCount()); + log.info(" Avg : {} (ms)", MetricX.getIteratorWaitAvg()); + log.info(" Max : {} (ms)", MetricX.getIteratorWaitMax()); + log.info(" Fail : {} (times)", metrics.getFailureCount()); + log.info(" Page : {} (KVs)", HgStoreClientConfig.of().getNetKvScannerPageSize()); + log.info("Iters : {}", iterators.size()); + log.info("*************************************************"); + + } + + @Test + public void benchmark_scanBatch2() throws IOException { + HgStoreTestUtil.println("--- Benchmark scanBatch2 ---"); + //String tableName = TABLE_NAME; + HgStoreSession session = getStoreSession(); + String keyName = "SCAN-BATCH"; + int keyAmt = 3000; + + Map data = + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, keyName, keyAmt, + //(key) -> toOwnerKey(key.substring(0, keyName.length() + // + 2), key) + (key) -> HgStoreTestUtil.toOwnerKey(0, key) + ); + + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME2, keyName, keyAmt, + //(key) -> toOwnerKey(key.substring(0, keyName.length() + 2), key) + (key) -> HgStoreTestUtil.toOwnerKey(0, key) + ); + + MetricX metrics = MetricX.ofStart(); + long t = System.currentTimeMillis(); + int count = 0; + String queryTable = null; + KvCloseableIterator> iterators = null; + List queryKeys = null; + List keys = new ArrayList<>(); + data.forEach((k, v) -> keys.add(k)); + + List prefixKeys = new ArrayList<>(); + prefixKeys.add(HgStoreTestUtil.toOwnerKey(0, keyName + "-0")); + prefixKeys.add(HgStoreTestUtil.toOwnerKey(0, keyName + "-1")); + prefixKeys.add(HgStoreTestUtil.toOwnerKey(0, keyName + "-2")); + prefixKeys.add(HgStoreTestUtil.toOwnerKey(0, keyName + "-3")); + + HgStoreTestUtil.println("-- test every key, one table --"); + count = 0; + queryTable = HgStoreTestUtil.TABLE_NAME; + queryKeys = keys; + iterators = session.scanBatch2( + HgScanQuery.prefixIteratorOf(queryTable, queryKeys.iterator(), + ScanOrderType.ORDER_NONE) + .builder() + .setScanType(0x40) + .build()); + while (iterators.hasNext()) { + HgKvIterator iterator = iterators.next(); + count += HgStoreTestUtil.amountOf(iterator); + } + iterators.close(); + Assert.assertEquals(keyAmt, count); + log.info(" size is {}", count); + + HgStoreTestUtil.println("-- test prefix key, one table --"); + count = 0; + queryTable = HgStoreTestUtil.TABLE_NAME; + queryKeys = prefixKeys; + iterators = session.scanBatch2( + HgScanQuery.prefixIteratorOf(queryTable, queryKeys.iterator(), + ScanOrderType.ORDER_STRICT) + .builder() + .setScanType(0x40) + .build()); + while (iterators.hasNext()) { + HgKvIterator iterator = iterators.next(); + count += HgStoreTestUtil.amountOf(iterator); + } + iterators.close(); + Assert.assertEquals(keyAmt, count); + log.info(" size is {}", count); + + HgStoreTestUtil.println("-- test prefix key, two table --"); + count = 0; + queryTable = HgStoreTestUtil.TABLE_NAME + "," + HgStoreTestUtil.TABLE_NAME2; + queryKeys = prefixKeys; + iterators = session.scanBatch2( + HgScanQuery.prefixIteratorOf(queryTable, queryKeys.iterator(), + ScanOrderType.ORDER_NONE) + .builder() + .setScanType(0x40) + .build()); + while (iterators.hasNext()) { + HgKvIterator iterator = iterators.next(); + count += HgStoreTestUtil.amountOf(iterator); + } + iterators.close(); + Assert.assertEquals(keyAmt * 2, count); + log.info(" size is {}", count); + + HgStoreTestUtil.println("-- test prefix key, two table, perKeyMax --"); + count = 0; + queryTable = HgStoreTestUtil.TABLE_NAME + "," + HgStoreTestUtil.TABLE_NAME2; + queryKeys = prefixKeys; + queryKeys.remove(queryKeys.size() - 1);//remove the last one. + iterators = session.scanBatch2( + HgScanQuery.prefixIteratorOf(queryTable, queryKeys.iterator(), + ScanOrderType.ORDER_NONE) + .builder() + .setScanType(0x40) + .setPerKeyMax(10) + .build()); + while (iterators.hasNext()) { + HgKvIterator iterator = iterators.next(); + count += HgStoreTestUtil.amountOf(iterator); + } + iterators.close(); + Assert.assertEquals(queryKeys.size() * 10, count); + log.info(" size is {}", count); + + keys.clear(); + + log.info("time is {}", System.currentTimeMillis() - t); + metrics.end(); + log.info("*************************************************"); + log.info("*********** Batch Scan Completed **************"); + log.info("Total : {} (sec)", metrics.past() / 1000); + log.info(" Sum : {} (sec)", MetricX.getIteratorWait() / 1000); + log.info(" Amt : {} (scans).", MetricX.getIteratorCount()); + log.info(" Avg : {} (ms)", MetricX.getIteratorWaitAvg()); + log.info(" Max : {} (ms)", MetricX.getIteratorWaitMax()); + log.info(" Fail : {} (times)", metrics.getFailureCount()); + log.info(" Page : {} (KVs)", HgStoreClientConfig.of().getNetKvScannerPageSize()); + log.info(" size is {}", count); + log.info("*************************************************"); + + + } + + // @Test + public void benchmark_scanBatch_SkipDegree() throws IOException { + HgStoreTestUtil.println("--- Benchmark scanBatch2 1Owner---"); + String tableName = HgStoreTestUtil.TABLE_NAME; + HgStoreSession session = getStoreSession(); + String keyName = "SCAN-BATCH"; + int keyAmt = 300000; + byte[] owner = "Owner".getBytes(); + Map data = + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt, key -> { + return HgStoreTestUtil.toOwnerKey(owner, key); + }); + HgStoreTestUtil.println("-- Starting scan --"); + MetricX metrics = MetricX.ofStart(); + //HgStoreTestUtil.println(session.scanIterator(tableName)); + long t = System.currentTimeMillis(); + int count = 0; + { + List keys = new ArrayList<>(); + keys.add(HgStoreTestUtil.toOwnerKey(owner, keyName)); + + KvCloseableIterator> iterators = + session.scanBatch2(HgScanQuery.prefixIteratorOf(tableName, keys.iterator(), + ScanOrderType.ORDER_NONE) + .builder().setScanType(0x40).setSkipDegree(1) + .build()); + //KvCloseableIterator> iterators = session.scanBatch2(HgScanQuery + // .tableOf(tableName)); + while (iterators.hasNext()) { + HgKvIterator iterator = iterators.next(); +// while (iterator.hasNext()){ +// System.out.println(new String(iterator.next().key())); +// } + count += HgStoreTestUtil.amountOf(iterator); + // log.info("{} - {}", new String(iterator.key()), iterator.value()); + } + keys.clear(); + iterators.close(); + iterators = null; + } + log.info("time is {}", System.currentTimeMillis() - t); + metrics.end(); + log.info("*************************************************"); + log.info("*********** Batch Scan Completed **************"); + log.info("Total : {} (sec)", metrics.past() / 1000); + log.info(" Sum : {} (sec)", MetricX.getIteratorWait() / 1000); + log.info(" Amt : {} (scans).", MetricX.getIteratorCount()); + log.info(" Avg : {} (ms)", MetricX.getIteratorWaitAvg()); + log.info(" Max : {} (ms)", MetricX.getIteratorWaitMax()); + log.info(" Fail : {} (times)", metrics.getFailureCount()); + log.info(" Page : {} (KVs)", HgStoreClientConfig.of().getNetKvScannerPageSize()); + log.info(" size is {}", count); + log.info("*************************************************"); + + + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/HgSessionManagerTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/HgSessionManagerTest.java new file mode 100644 index 0000000000..9a63ce74ff --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/HgSessionManagerTest.java @@ -0,0 +1,1125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store; + +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.ALL_PARTITION_OWNER; +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +import org.apache.hugegraph.pd.common.PartitionUtils; +import org.apache.hugegraph.store.client.HgStoreNodeManager; +import org.apache.hugegraph.store.client.util.ExecutorPool; +import org.apache.hugegraph.store.client.util.HgStoreClientConfig; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.client.util.MetricX; +import org.apache.hugegraph.store.util.HgStoreTestUtil; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class HgSessionManagerTest { + private static final Map leaderMap = new ConcurrentHashMap<>(); + private static final Map storeMap = new ConcurrentHashMap<>(); + + private static final ExecutorService pool = Executors.newFixedThreadPool(100, + ExecutorPool.newThreadFactory( + "unit-test")); + + private static final int partitionCount = 10; + // 需要与 store 的 application.yml 的 fake-pd.partition-count 保持一致 + + //private static String[] storeAddress = {"127.0.0.1:8500"}; + private static final String[] storeAddress = + {"127.0.0.1:8501", "127.0.0.1:8502", "127.0.0.1:8503"}; + + private static final int PARTITION_LENGTH = getPartitionLength(); + + private static int getPartitionLength() { + return PartitionUtils.MAX_VALUE / + (partitionCount == 0 ? storeAddress.length : partitionCount) + 1; + } + + @BeforeClass + public static void init() { + for (String address : storeAddress) { + storeMap.put((long) address.hashCode(), address); + } + for (int i = 0; i < partitionCount; i++) { + leaderMap.put(i, storeMap.keySet().iterator().next()); + } + + HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance(); + nodeManager.setNodePartitioner((builder, graphName, startKey, endKey) -> { + int startCode = PartitionUtils.calcHashcode(startKey); + int endCode = PartitionUtils.calcHashcode(endKey); + if (ALL_PARTITION_OWNER == startKey) { + storeMap.forEach((k, v) -> { + builder.add(k, -1); + }); + } else if (endKey == HgStoreClientConst.EMPTY_BYTES || startKey == endKey || + Arrays.equals(startKey, endKey)) { + //log.info("leader-> {}",leaderMap.get(startCode / PARTITION_LENGTH)); + builder.add(leaderMap.get(startCode / PARTITION_LENGTH), startCode); + } else { + Assert.fail("OwnerKey 转成 HashCode 后已经无序了，按照 OwnerKey 范围查询没意义"); + builder.add(leaderMap.get(startCode / PARTITION_LENGTH), startCode); + builder.add(leaderMap.get(endCode / PARTITION_LENGTH), endCode); + } + return 0; + }); + nodeManager.setNodeProvider((graphName, nodeId) -> { + // System.out.println("HgStoreNodeProvider apply " + graphName + " " + nodeId + + // " " + storeMap.get(nodeId)); + return nodeManager.getNodeBuilder().setNodeId(nodeId) + .setAddress(storeMap.get(nodeId)).build(); + }); + nodeManager.setNodeNotifier((graphName, storeNotice) -> { + System.out.println("recv node notifier " + storeNotice); + if (storeNotice.getPartitionLeaders().size() > 0) { + leaderMap.putAll(storeNotice.getPartitionLeaders()); + System.out.println("leader changed "); + leaderMap.forEach((k, v) -> { + System.out.print(" " + k + " " + v + ","); + }); + System.out.println(); + } + return 0; + }); + } + + protected static HgStoreSession getStoreSession() { + return HgSessionManager.getInstance().openSession(HgStoreTestUtil.GRAPH_NAME); + } + + protected static HgStoreSession getStoreSession(String graph) { + return HgSessionManager.getInstance().openSession(graph); + } + + @Test + public void put_get() { + System.out.println("--- test put & get ---"); + + HgStoreSession session = getStoreSession(); + + HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY"); + //HgOwnerKey key = toAllNodeKey("FOR-PUT-KEY"); + byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); + + Assert.assertTrue(session.put(HgStoreTestUtil.TABLE_NAME, key, value)); + Assert.assertEquals(HgStoreTestUtil.toStr(value), HgStoreTestUtil.toStr(session.get( + HgStoreTestUtil.TABLE_NAME, key))); + } + + @Test + public void batchPrefix() { + System.out.println("--- test batchGet ---"); + HgStoreSession session = getStoreSession(); + String keyPrefix = "UNIT-BATCH-GET"; + + Map map = HgStoreTestUtil.batchPut(session, keyPrefix); + List keyList = + map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()); + + Map keyMap = new HashMap<>(); + map.entrySet().forEach(e -> { + keyMap.put(Arrays.toString(e.getKey().getKey()), e.getValue()); + }); + + //printOwner(keyList); + HgKvIterator iterator = session.batchPrefix(HgStoreTestUtil.TABLE_NAME, keyList); + int amount = HgStoreTestUtil.println(iterator); + Assert.assertEquals(amount, keyList.size()); +/* + + println("--- batch-get result ---"); + iterator = session.batchGet(TABLE_NAME, keyList); + while(iterator.hasNext()){ + HgKvEntry kv=iterator.next(); + Assert.assertEquals(true,keyMap.containsKey(Arrays.toString(kv.key()))); + } +*/ + + } + + @Test + //CAUTION: ONLY FOR LONG！ + //注意：目前只可以对 long 类型 value 进行 Merge 操作。 + public void merge() { + System.out.println("--- test merge (1+1=2)---"); + HgStoreSession session = getStoreSession(); + String mergeKey = "merge-key"; + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(mergeKey); + byte[] value = HgStoreTestUtil.toBytes(1L); + + System.out.println("- put " + mergeKey + ":1 -"); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); + System.out.println("- merge " + mergeKey + ":1 -"); + session.merge(HgStoreTestUtil.TABLE_NAME, key, value); + long res = HgStoreTestUtil.toLong(session.get(HgStoreTestUtil.TABLE_NAME, key)); + System.out.printf("after merge " + mergeKey + "=%s%n", res); + Assert.assertEquals(2L, res); + + String putStr = "19"; + session.put(HgStoreTestUtil.TABLE_NAME, key, HgStoreTestUtil.toBytes(putStr)); + byte[] b1 = session.get(HgStoreTestUtil.TABLE_NAME, key); + Assert.assertEquals(putStr, HgStoreTestUtil.toStr(b1)); + } + + @Test + public void delete() { + System.out.println("--- test delete ---"); + HgStoreSession session = getStoreSession(); + + String delKey = "del-key"; + String delValue = "del-value"; + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); + byte[] value = HgStoreTestUtil.toBytes(delValue); + + HgStoreTestUtil.println("- put " + delKey + ": " + delValue); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); + HgStoreTestUtil.println("- delete " + delKey); + session.delete(HgStoreTestUtil.TABLE_NAME, key); + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println("- get " + delKey + ": " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals(EMPTY_BYTES, value); + } + + // @Test + public void deleteSingle() { + System.out.println("--- test deleteSingle ---"); + HgStoreSession session = getStoreSession(); + + String delKey = "del-single-key"; + String delValue = "del-single-value"; + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); + byte[] value = HgStoreTestUtil.toBytes(delValue); + + HgStoreTestUtil.println("- put [" + delKey + "] = " + delValue); + session.put(HgStoreTestUtil.TABLE_NAME, key, value); + + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println( + "- before del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals(delValue, HgStoreTestUtil.toStr(value)); + + HgStoreTestUtil.println("- delete-single : [" + delKey + "]"); + session.deleteSingle(HgStoreTestUtil.TABLE_NAME, key); + value = session.get(HgStoreTestUtil.TABLE_NAME, key); + HgStoreTestUtil.println( + "- after del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals("", HgStoreTestUtil.toStr(value)); + + } + + // @Test + public void deleteRange() { + HgStoreTestUtil.println("--- test deleteRange ---"); + HgStoreSession session = getStoreSession(); + + String rangePrefix = "DEL-RANGE-KEY"; + String owner = "batch-delete-owner"; + Map map = + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, rangePrefix, 10, + key -> HgStoreTestUtil.toOwnerKey(owner, key) + ); + + HgOwnerKey startKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-00"); + HgOwnerKey endKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-05"); + + Assert.assertTrue(session.deleteRange(HgStoreTestUtil.TABLE_NAME, startKey, endKey)); + + HgStoreTestUtil.println("- after delete range from [" + + HgStoreTestUtil.toStr(startKey.getKey()) + "] to [" + + HgStoreTestUtil.toStr(endKey.getKey()) + "]"); + + for (int i = 0; i < 10; i++) { + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-" + + HgStoreTestUtil.padLeftZeros( + String.valueOf(i), 2)); + String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key)); + HgStoreTestUtil.println( + "- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value); + + // TODO: [start,end)? + if (i < 5) { + Assert.assertEquals("", value); + } else { + //println(value); + Assert.assertNotEquals("", value); + } + } + + } + + // @Test + public void deletePrefix() { + System.out.println("--- test deletePrefix ---"); + HgStoreSession session = getStoreSession(); + + String prefixStr = "DEL-PREFIX-KEY"; + String owner = "batch-delete-owner"; + Map map = + HgStoreTestUtil.batchPut(session, HgStoreTestUtil.TABLE_NAME, prefixStr, 10, + key -> { + return HgStoreTestUtil.toOwnerKey(owner, key); + }); + + //printOwner(map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList())); + + HgOwnerKey prefix = HgStoreTestUtil.toOwnerKey(owner, prefixStr); + + Assert.assertEquals(10, HgStoreTestUtil.amountOf(session.scanIterator( + HgStoreTestUtil.TABLE_NAME, prefix))); + session.deletePrefix(HgStoreTestUtil.TABLE_NAME, prefix); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator( + HgStoreTestUtil.TABLE_NAME, prefix))); + + HgStoreTestUtil.println("- after delete by prefix:[" + prefixStr + "]"); + + for (int i = 0; i < 10; i++) { + HgOwnerKey key = + HgStoreTestUtil.toOwnerKey(owner, prefixStr + HgStoreTestUtil.toSuffix(i, 2)); + String value = HgStoreTestUtil.toStr(session.get(HgStoreTestUtil.TABLE_NAME, key)); + System.out.println("- get [" + HgStoreTestUtil.toStr(key.getKey()) + "] = " + value); + Assert.assertEquals("", value); + } + + } + + // @Test + public void scan() { + HgStoreTestUtil.println("--- test scanIterator ---"); + String tableName = "UNIT_SCAN_ITERATOR"; + String keyName = "SCAN-ITER"; + int keyAmt = 100; + + HgStoreSession session = getStoreSession(); + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 10)) < + 10) { + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); + } + int count = 0; + int limit = 0; + int max = 99999; + HgKvIterator iterator = null; + + // println("-- test scan all --"); + iterator = session.scanIterator(tableName); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 10 == 0) { + // println(entry); + } + if (count >= max) break; + } + iterator.close(); + Assert.assertEquals(keyAmt, count); + + // println("-- test 0 element --"); + iterator = session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey("__SCAN-001"), + HgStoreTestUtil.toAllPartitionKey("__SCAN-100"), 0); + Assert.assertFalse(iterator.hasNext()); + try { + iterator.next(); + Assert.fail(); + } catch (Throwable t) { + // println("-- test NoSuchElementException --"); + Assert.assertTrue(t instanceof NoSuchElementException); + } + iterator.close(); + + // println("-- test limit 1 to 10 --"); + for (int i = 1; i <= 10; i++) { + // println("- limit " + i + " -"); + limit = i; + iterator = session.scanIterator(tableName, + HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), + limit); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + // println(entry); + } + iterator.close(); + Assert.assertEquals(limit, count); + } + + // println("-- test limit 1 to 10 not enough --"); + for (int i = 1; i <= 10; i++) { + // println("- limit " + i + " -"); + limit = i; + iterator = session.scanIterator(tableName + , HgStoreTestUtil.toAllPartitionKey(keyName + "-001") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-005"), limit); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + // println(entry); + } + iterator.close(); + if (i <= 5) { + Assert.assertEquals(limit, count); + } else { + Assert.assertEquals(5, count); + } + + } + + // println("-- test limit 0 (no limit) --"); + limit = 0; + iterator = + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-2"), limit); + + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 10 == 0) { + // println(entry); + } + if (count >= max) break; + } + iterator.close(); + Assert.assertEquals(keyAmt, count); + + // println("-- test scan prefix --"); + iterator = + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-")); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 10 == 0) { + // println(entry); + } + if (count >= max) break; + } + iterator.close(); + Assert.assertEquals(keyAmt, count); + + // println("-- test scan via hash code --"); + iterator = session.scanIterator(tableName, 0, 65535, HgKvStore.SCAN_HASHCODE, EMPTY_BYTES); + count = 0; + + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 10 == 0) { + // println(entry); + } + if (count >= max) break; + } + iterator.close(); + Assert.assertEquals(keyAmt, count); + + // println("-- test range limit scan type --"); + iterator = session.scanIterator(tableName + , HgStoreTestUtil.toAllPartitionKey("WWWWWWW") + , HgStoreClientConst.EMPTY_OWNER_KEY + , 0, HgKvStore.SCAN_ANY, EMPTY_BYTES + ); + iterator.close(); + // println("-- test range limit scan type -session.scanIterator over--"); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(iterator)); + // println("-- test range limit scan type -session.assertEquals over--"); + + // println("-- test range limit scan type -100"); + iterator = session.scanIterator(tableName + , HgStoreTestUtil.toAllPartitionKey(keyName + "-100") + , HgStoreClientConst.EMPTY_OWNER_KEY + , 0, HgKvStore.SCAN_ANY, EMPTY_BYTES + ); + + // println("-- test range limit scan type -100 over"); + Assert.assertEquals(1, HgStoreTestUtil.amountOf(iterator)); + // println("-- test range limit scan type -100 assertEquals 1"); + + // println("-- test range limit scan type -51 "); + iterator = session.scanIterator(tableName + , HgStoreTestUtil.toAllPartitionKey(keyName + "-051") + , HgStoreClientConst.EMPTY_OWNER_KEY + ); + + // println("-- test range limit scan type -51 over"); + //HgStoreTestUtil.println(iterator); + Assert.assertEquals(50, HgStoreTestUtil.amountOf(iterator)); + // println("-- test range limit scan type -51 assertEquals"); + //TODO: add more... + HgStoreTestUtil.println("-- test scanIterator end"); + } + + // @Test + public void scan_close() { + /*************** test scan close **************/ + HgStoreTestUtil.println("--- test scan close ---"); + String tableName = "UNIT_ITER_CLOSE"; + String keyName = "UNIT_ITER_CLOSE"; + + int amount = 1_000_000; + HgStoreSession session = getStoreSession(); + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) < + 1) { + HgStoreTestUtil.batchPut(session, tableName, keyName, amount); + } + + HgKvIterator iterator = null; + + /* server over, all -> close */ + iterator = session.scanIterator(tableName); + Assert.assertEquals(amount, HgStoreTestUtil.amountOf(iterator)); + + /* server over, one page, all -> close */ + iterator = session.scanIterator(tableName, 100_000); + Assert.assertEquals(100_000, HgStoreTestUtil.amountOf(iterator)); + + /* server over, two page, all -> close */ + iterator = session.scanIterator(tableName, 200_000); + Assert.assertEquals(200_000, HgStoreTestUtil.amountOf(iterator)); + + /* server not over, enough -> close */ + iterator = session.scanIterator(tableName); + iterator.next(); + iterator.close(); + + /* server not over, one page, enough -> close */ + iterator = session.scanIterator(tableName, 100_000); + iterator.next(); + iterator.close(); + + /* server not over, two page, enough -> close */ + iterator = session.scanIterator(tableName, 200_000); + iterator.next(); + iterator.close(); + + /* server not over, enough -> close */ + iterator = session.scanIterator(tableName); + iterator.next(); + iterator.close(); + iterator = session.scanIterator(tableName); + for (int i = 0; iterator.hasNext() && i < 500_000; i++) { + iterator.next(); + } + iterator.close(); + + /* server timeout, enough -> close */ + iterator = session.scanIterator(tableName); + iterator.next(); + // HgStoreTestUtil.sleeping(11000); + + try { + HgStoreTestUtil.amountOf(iterator); + } catch (Throwable t) { + HgStoreTestUtil.println("-- passed server waiting timeout --"); + } + + iterator.close(); + + } + + // @Test + public void paging() { + HgStoreTestUtil.println("--- test scanIterator_range ---"); + String graph = "UNIT/paging"; + String tableName = "UNIT_SCAN_PAGING"; + String keyName = "SCAN-PAGING"; + int keyAmt = 100; + HgStoreSession session = getStoreSession(graph); + + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) < + 1) { + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); + } + + int count = 0; + int limit = 0; + int max = 99999; + HgKvIterator iterator = null; + + count = 0; + + iterator = session.scanIterator(tableName + , HgStoreTestUtil.toAllPartitionKey(keyName + "-000") + , HgStoreClientConst.EMPTY_OWNER_KEY + , 0, HgKvStore.SCAN_ANY, EMPTY_BYTES + ); + //HgStoreTestUtil.println(iterator); + // Assert.assertEquals(100, HgStoreTestUtil.amountOf(iterator)); + List positionList = new LinkedList<>(); + while (iterator.hasNext()) { + HgStoreTestUtil.print((count++) + " "); + HgKvEntry entry = iterator.next(); + HgStoreTestUtil.print(entry); + HgStoreTestUtil.print(" " + Arrays.toString(iterator.position()) + "\n"); + positionList.add(iterator.position()); + if (count >= max) break; + } + + + iterator = session.scanIterator(tableName + , HgStoreTestUtil.toAllPartitionKey(keyName + "-000") + , HgStoreClientConst.EMPTY_OWNER_KEY + , 0, HgKvStore.SCAN_ANY, EMPTY_BYTES + ); + + byte[] position = positionList.get(50); + HgStoreTestUtil.println("seek: " + Arrays.toString(position)); + iterator.seek(position); + //println("amt after seek: "+HgStoreTestUtil.println(iterator)); + Assert.assertEquals(50, HgStoreTestUtil.amountOf(iterator)); + + } + + @Test + public void scanBatch() { + HgStoreTestUtil.println("--- test scanBatch ---"); + String tableName = "UNIT_SCAN_BATCH_1"; + String keyName = "SCAN-BATCH"; + int keyAmt = 10_000; + + HgStoreSession session = getStoreSession(); + + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), + keyAmt)) < keyAmt) { + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); + } + + Assert.assertEquals(keyAmt, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + + List> iterators = null; + List prefixes = null; + + List startList = Arrays.asList( + HgStoreTestUtil.toAllPartitionKey(keyName + "-001") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-003") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-005") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-007") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-009") + ); + + List endList = Arrays.asList( + HgStoreTestUtil.toAllPartitionKey(keyName + "-002") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-004") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-006") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-008") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-010") + ); + + List prefixList = Arrays.asList( + HgStoreTestUtil.toAllPartitionKey(keyName + "-001") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-002") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-003") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-004") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-005") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-006") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-007") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-008") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-009") + ); + + HgStoreTestUtil.println("-- test scan-batch all --"); + + HgScanQuery scanQuery = HgScanQuery.tableOf(tableName); + iterators = session.scanBatch(scanQuery); + Assert.assertEquals(3, iterators.size()); + + Assert.assertEquals(keyAmt, HgStoreTestUtil.println(iterators)); + + HgStoreTestUtil.println("-- test scan-batch prefix --"); + + + iterators = session.scanBatch( + HgScanQuery.prefixOf(tableName, prefixList) + ); + Assert.assertEquals(3, iterators.size()); + Assert.assertEquals(900, + iterators.parallelStream().mapToInt(e -> HgStoreTestUtil.amountOf(e)) + .sum() + ); + + HgStoreTestUtil.println("-- test scan-batch range --"); + + iterators = session.scanBatch(HgScanQuery.rangeOf(tableName, startList, endList)); + Assert.assertEquals(3, iterators.size()); + Assert.assertEquals( + HgStoreTestUtil.amountOf( + session.scanIterator(tableName + , HgStoreTestUtil.toAllPartitionKey(keyName + "-001") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-010") + ) + ) + , + iterators.parallelStream().mapToInt(e -> HgStoreTestUtil.amountOf(e)).sum() + ); + + HgStoreTestUtil.println("-- test scan-batch limit --"); + + int limit = 1; + iterators = session.scanBatch( + HgScanQuery.rangeOf(tableName, startList, endList) + .builder() + .setLimit(limit) + .build() + ); + + //HgStoreTestUtil.println(iterators); + Assert.assertEquals(iterators.size() * limit, + iterators.parallelStream().mapToInt(e -> HgStoreTestUtil.amountOf(e)) + .sum() + ); + + HgStoreTestUtil.println("-- test scan-batch multi-table --"); + if (HgStoreTestUtil.amountOf( + session.scanIterator("g+oe", HgStoreTestUtil.toAllPartitionKey(keyName), keyAmt)) < + keyAmt) { + HgStoreTestUtil.batchPut(session, "g+oe", keyName, keyAmt); + HgStoreTestUtil.batchPut(session, "g+ie", keyName, keyAmt); + } + + prefixes = Collections.singletonList(HgStoreTestUtil.toAllPartitionKey(keyName)); + + tableName = "g+oe,g+ie"; + iterators = session.scanBatch( + HgScanQuery.prefixOf(tableName, prefixes) + ); + + //HgStoreTestUtil.println(iterators); + Assert.assertEquals(keyAmt * 2, HgStoreTestUtil.amountIn(iterators)); + iterators = session.scanBatch( + HgScanQuery.prefixOf(tableName, prefixList) + ); + Assert.assertEquals(900 * 2, HgStoreTestUtil.amountIn(iterators)); + + HgStoreTestUtil.println("-- test scan-batch per-key-limit --"); + + tableName = "PER_KEY_LIMIT_TABLE"; + keyName = "PER_KEY_LIMIT"; + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), + keyAmt)) < keyAmt) { + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); + } + prefixes = Arrays.asList( + HgStoreTestUtil.toAllPartitionKey(keyName + "-01") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-02") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-03") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-04") + , HgStoreTestUtil.toAllPartitionKey(keyName + "-05") + ); + + iterators = session.scanBatch( + HgScanQuery.prefixOf(tableName, prefixes).builder().setPerKeyLimit(1).build() + ); + //HgStoreTestUtil.println(iterators); + Assert.assertEquals(prefixes.size() * 3, HgStoreTestUtil.amountIn(iterators)); + + } + + // @Test + public void truncate() { + HgStoreTestUtil.println("--- test truncate ---"); + String graph = "graph_truncate"; + String tableName = "UNIT_TRUNCATE_1"; + String keyName = "KEY_TRUNCATE"; + + HgStoreSession session = getStoreSession(graph); + HgStoreTestUtil.batchPut(session, tableName, keyName, 100); + Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + + String tableName2 = "UNIT_TRUNCATE_2"; + HgStoreTestUtil.batchPut(session, tableName2, keyName, 100); + Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); + + session.truncate(); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); + } + + // @Test + public void handle_table() { + HgStoreTestUtil.println("--- test table ---"); + + String tableName = "UNIT_TABLE_" + System.currentTimeMillis(); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY"); + byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); + HgStoreSession session = getStoreSession(); + + HgStoreTestUtil.println("-- test createTable --"); + session.createTable(tableName); + Assert.assertTrue(session.existsTable(tableName)); + + HgStoreTestUtil.println("-- test deleteTable --"); + session.put(tableName, key, value); + Assert.assertEquals(HgStoreTestUtil.toStr(value), + HgStoreTestUtil.toStr(session.get(tableName, key))); + session.deleteTable(tableName); + Assert.assertNotEquals(HgStoreTestUtil.toStr(value), + HgStoreTestUtil.toStr(session.get(tableName, key))); + Assert.assertTrue(session.existsTable(tableName)); + + + HgStoreTestUtil.println("-- test dropTable --"); + Assert.assertTrue(session.dropTable(tableName)); + Assert.assertFalse(session.existsTable(tableName)); + + + HgStoreTestUtil.println("-- test existsTable --"); + Assert.assertFalse(session.existsTable(tableName)); + + } + + // @Test + public void tx() { + HgStoreTestUtil.println("--- test tx ---"); + HgStoreSession session = getStoreSession(); + String tableName = "UNIT_TABLE_TX"; + String keyPrefix = "TX"; + int keyAmt = 100; + + HgStoreTestUtil.println("-- unsupported tx operation --"); + HgStoreTestUtil.println("- tx deleteTable -"); + HgStoreTestUtil.batchPut(session, tableName, keyPrefix, keyAmt); + Assert.assertEquals(keyAmt, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + session.beginTx(); + session.deleteTable(tableName); + session.commit(); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + + HgStoreTestUtil.println("- tx dropTable -"); + HgStoreTestUtil.batchPut(session, tableName, keyPrefix, keyAmt); + Assert.assertEquals(keyAmt, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + session.beginTx(); + session.dropTable(tableName); + session.commit(); + Assert.assertFalse(session.existsTable(tableName)); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + + HgStoreTestUtil.println("- tx truncate -"); + HgStoreTestUtil.batchPut(session, tableName, keyPrefix, keyAmt); + Assert.assertEquals(keyAmt, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + session.beginTx(); + session.truncate(); + session.commit(); + Assert.assertFalse(session.existsTable(tableName)); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + + //TODO : add others + } + + //// @Test + public void scanIterator_WithNonePartition() { + HgStoreTestUtil.println("--- test scanIterator with none partition ---"); + int count = 0; + HgStoreSession session = getStoreSession(); + + for (int i = 0; i < 100; i++) { + HgKvIterator iterator = session.scanIterator("XXXXXXXX"); + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + HgStoreTestUtil.println(entry); + } + } + + Assert.assertEquals(0, count); + + } + + //// @Test + public void repeatedly_parallel_scan() { + MetricX metrics = MetricX.ofStart(); + HgStoreTestUtil.repeatedlyTest(100, () -> parallel_scan()); + metrics.end(); + + log.info("*************************************************"); + log.info("************* Scanning Completed **************"); + log.info("Total: {} sec.", metrics.past() / 1000); + log.info("Sum: {} sec.", MetricX.getIteratorWait() / 1000); + log.info("Amt: {} scans.", MetricX.getIteratorCount()); + log.info("Avg: {} ms.", MetricX.getIteratorWaitAvg()); + log.info("Max: {} ms.", MetricX.getIteratorWaitMax()); + log.info("Fail: {} times.", metrics.getFailureCount()); + log.info("Page: {}", HgStoreClientConfig.of().getNetKvScannerPageSize()); + log.info("*************************************************"); + + // runWaiting(); + } + + // @Test + public void parallel_scan() { + HgStoreTestUtil.println("--- test scanIterator in parallel ---"); + HgStoreTestUtil.parallelTest(100, () -> { + this.scan(); + }, t -> t.printStackTrace()); + } + + //// @Test + public void put_Benchmark() { + /*************** Put Benchmark **************/ + String tableName = "UNIT_PUT_BENCHMARK"; + String keyPrefix = "PUT-BENCHMARK"; + int amount = 1_000_000; + HgStoreSession session = getStoreSession(); + + int length = String.valueOf(amount).length(); + + session.beginTx(); + + long start = System.currentTimeMillis(); + for (int i = 0; i < amount; i++) { + HgOwnerKey key = HgStoreTestUtil.toOwnerKey( + keyPrefix + "-" + HgStoreTestUtil.padLeftZeros(String.valueOf(i), length)); + byte[] value = HgStoreTestUtil.toBytes(keyPrefix + "-V-" + i); + + session.put(tableName, key, value); + + if ((i + 1) % 100_000 == 0) { + HgStoreTestUtil.println("---------- " + (i + 1) + " --------"); + HgStoreTestUtil.println( + "Preparing took: " + (System.currentTimeMillis() - start) + " ms."); + session.commit(); + HgStoreTestUtil.println( + "Committing took: " + (System.currentTimeMillis() - start) + " ms."); + start = System.currentTimeMillis(); + session.beginTx(); + } + } + + if (session.isTx()) { + session.commit(); + } + + Assert.assertEquals(amount, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + + + } + + //// @Test + public void put_Benchmark_Parallel() { + int threadsAmount = 100; + CountDownLatch countDownLatch = new CountDownLatch(threadsAmount); + + for (int j = 0; j < threadsAmount; j++) { + for (int i = 0; i < threadsAmount; i++) { + pool.submit(() -> { + this.put_Benchmark(); + // this.scanIterator_WithNonePartition(); + countDownLatch.countDown(); + }); + } + } + try { + countDownLatch.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + } + + //// @Test + public void parallel_scanBatch() { + int threadsAmount = 50; + CountDownLatch countDownLatch = new CountDownLatch(threadsAmount); + + for (int j = 0; j < threadsAmount; j++) { + for (int i = 0; i < threadsAmount; i++) { + pool.submit(() -> { + try { + this.scanBatch(); + } catch (Throwable t) { + t.printStackTrace(); + } finally { + countDownLatch.countDown(); + } + }); + } + } + try { + countDownLatch.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + } + + //// @Test + public void benchmark_scanBatch() { + HgStoreTestUtil.println("--- Benchmark scanBatch ---"); + String tableName = "Benchmark_SCAN_BATCH"; + String keyName = "SCAN-BATCH"; + int keyAmt = 10_000_000; + + HgStoreSession session = getStoreSession(); + + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) < + 1) { + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); + } + HgStoreTestUtil.println("-- Starting scan --"); + // while (true) { + + MetricX metrics = MetricX.ofStart(); + //HgStoreTestUtil.println(session.scanIterator(tableName)); + List> iterators = session.scanBatch(HgScanQuery.tableOf(tableName)); + //HgStoreTestUtil.println(iterators); + //Assert.assertEquals(keyAmt, HgStoreTestUtil.amountIn(iterators)); + HgStoreTestUtil.amountIn(iterators); + metrics.end(); + log.info("*************************************************"); + log.info("*********** Batch Scan Completed **************"); + log.info("Total : {} (sec)", metrics.past() / 1000); + log.info(" Sum : {} (sec)", MetricX.getIteratorWait() / 1000); + log.info(" Amt : {} (scans).", MetricX.getIteratorCount()); + log.info(" Avg : {} (ms)", MetricX.getIteratorWaitAvg()); + log.info(" Max : {} (ms)", MetricX.getIteratorWaitMax()); + log.info(" Fail : {} (times)", metrics.getFailureCount()); + log.info(" Page : {} (KVs)", HgStoreClientConfig.of().getNetKvScannerPageSize()); + log.info("Iters : {}", iterators.size()); + log.info("*************************************************"); + HgStoreTestUtil.sleeping(100); + + // } + } + + //// @Test + public void benchmark_scan() { + /*************** test no limit, with 10 millions **************/ + String tableName = "UNIT_HUGE"; + String keyName = "SCAN-HUGE"; + int amount = 10_000_000; + int max = 10_000_000; + HgStoreSession session = getStoreSession(); + + /*Initialization*/ + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) < + 1) { + HgStoreTestUtil.batchPut(session, tableName, keyName, amount); + } + + MetricX metricX = MetricX.ofStart(); + + int count = 0; + HgKvIterator iterator = session.scanIterator(tableName); + //HgStoreTestUtil.println(iterator, e -> (e % (amount / 100) == 0)); + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % (amount / 10) == 0) { + HgStoreTestUtil.println(entry); + } + if (count >= max) break; + } + + metricX.end(); + + log.info("*************************************************"); + log.info("************* Benchmark Completed *************"); + log.info("Keys: {}", count); + log.info("Waiting: {} ms.", MetricX.getIteratorWait()); + log.info("Total: {} ms.", metricX.past()); + log.info("Iterator: [{}]", iterator.getClass().getSimpleName()); + log.info("Page: {}", HgStoreClientConfig.of().getNetKvScannerPageSize()); + log.info("*************************************************"); + + Assert.assertEquals(amount, count); + } + + + //// @Test + public void extreme_scan_close() { + /*************** test close **************/ + String tableName = "UNIT_ITER_CLOSE_EXTREME"; + String keyName = "UNIT_ITER_CLOSE_EXTREME"; + int amount = 1_000_000; + HgStoreSession session = getStoreSession(); + + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) < + 1) { + HgStoreTestUtil.batchPut(session, tableName, keyName, amount); + } + HgKvIterator iterator = null; + /* server not over, enough, extreme -> close */ + for (int i = 0; i <= 10_000; i++) { + iterator = session.scanIterator(tableName); + iterator.next(); + iterator.close(); + HgStoreTestUtil.println("extreme loop: " + i); + } + HgStoreTestUtil.runWaiting(); + } + + //// @Test + public void parallel_scan_close() { + HgStoreTestUtil.parallelTest(10, () -> this.scan_close(), t -> t.printStackTrace()); + } + + //// @Test + public void repeat_parallel_scan_close() { + HgStoreTestUtil.repeatedlyTest(1000, () -> this.parallel_scan_close()); + HgStoreTestUtil.runWaiting(); + } + + //// @Test + public void parallel_huge_scan() { + int threadsAmount = 3; + CountDownLatch countDownLatch = new CountDownLatch(threadsAmount); + ExecutorService poll = Executors.newFixedThreadPool(threadsAmount); + + for (int i = 0; i < threadsAmount; i++) { + poll.submit(() -> { + this.benchmark_scan(); + countDownLatch.countDown(); + }); + } + + try { + countDownLatch.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + } + +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/PartitionEngineTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/PartitionEngineTest.java new file mode 100644 index 0000000000..7115aaa65d --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/PartitionEngineTest.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.store.cmd.HgCmdClient; +import org.apache.hugegraph.store.meta.Partition; +import org.junit.Assert; + +import com.alipay.sofa.jraft.option.RpcOptions; + +public class PartitionEngineTest { + + // @Test + public void test() { + String[] peers = new String[]{"1", "2", "6"}; + String[] learners = new String[]{"3", "7"}; + List oldPeers1 = Arrays.asList("1", "2", "3"); + List oldLearner1 = Arrays.asList("4", "5"); + + + List oldPeers = + oldPeers1.stream().map(peer -> peer).collect(Collectors.toList()); + oldPeers.addAll( + oldLearner1.stream().map(peer -> peer).collect(Collectors.toList())); + + List addedNodes = Arrays.stream(peers).filter(peer -> + !oldPeers.stream() + .map(e -> e) + .collect( + Collectors.toSet()) + .contains(peer)) + .collect(Collectors.toList()); + addedNodes.addAll(Arrays.stream(learners).filter(peer -> + !oldPeers.stream() + .map(e -> e) + .collect( + Collectors.toSet()) + .contains(peer)) + .collect(Collectors.toList())); + + List removedNodes = oldPeers.stream().filter(peer -> + !Arrays.stream(peers) + .map(e -> e).collect( + Collectors.toSet()) + .contains(peer)) + .collect(Collectors.toList()); + removedNodes = removedNodes.stream().filter(peer -> + !Arrays.stream(learners).map(e -> e) + .collect(Collectors.toSet()) + .contains(peer)) + .collect(Collectors.toList()); + List mixedPeer = oldPeers1.stream().filter(peer -> + Arrays.stream(learners) + .map(e -> e).collect( + Collectors.toSet()) + .contains(peer)) + .collect(Collectors.toList()); + + // 新增 6、7 + Assert.assertEquals(2, addedNodes.size()); + addedNodes.clear(); + addedNodes.addAll(Arrays.asList(peers)); + addedNodes.addAll(Arrays.asList(learners)); + addedNodes.removeAll(oldPeers); + Assert.assertEquals(2, addedNodes.size()); + + addedNodes.forEach(s -> System.out.print(s + " ")); + System.out.println(); + // 删除 4，5 + Assert.assertEquals(2, removedNodes.size()); + + removedNodes.clear(); + removedNodes.addAll(oldPeers); + removedNodes.removeAll(Arrays.asList(peers)); + removedNodes.removeAll(Arrays.asList(learners)); + Assert.assertEquals(2, removedNodes.size()); + removedNodes.forEach(s -> System.out.print(s + " ")); + System.out.println(); + // 交集 5 + Assert.assertEquals(1, mixedPeer.size()); + oldPeers1.removeAll(Arrays.asList(learners)); + Assert.assertEquals(1, oldPeers1.size()); + mixedPeer.forEach(s -> System.out.print(s + " ")); + + } + + + // @Test + public void testPartition() { + Partition p = new Partition(); + p.setId(1); + List shards = new ArrayList<>(); + shards.add(Metapb.Shard.newBuilder().build()); + // p.setShardsList(shards); + + Partition p2 = p.clone(); + p.setId(2); + Assert.assertNotEquals(p2.getId(), p.getId()); + + } + + // @Test + public void testUpdateShardsList() { + List curShards = new ArrayList<>(); + curShards.add(Metapb.Shard.newBuilder().setStoreId(1001).setRole(Metapb.ShardRole.Follower) + .build()); + curShards.add(Metapb.Shard.newBuilder().setStoreId(1002).setRole(Metapb.ShardRole.Leader) + .build()); + curShards.add(Metapb.Shard.newBuilder().setStoreId(1003).setRole(Metapb.ShardRole.Follower) + .build()); + + List reqShards = new ArrayList<>(); + reqShards.add(Metapb.Shard.newBuilder().setStoreId(1001).setRole(Metapb.ShardRole.Leader) + .build()); + reqShards.add(Metapb.Shard.newBuilder().setStoreId(1002).setRole(Metapb.ShardRole.Leader) + .build()); + reqShards.add(Metapb.Shard.newBuilder().setStoreId(1004).setRole(Metapb.ShardRole.Leader) + .build()); + + + long leaderStoreId = 0; + for (Metapb.Shard shard : curShards) { + if (shard.getRole() == Metapb.ShardRole.Leader) { + leaderStoreId = shard.getStoreId(); + break; + } + } + + // remove + List shards = curShards.stream().filter(shard -> + reqShards.stream() + .map(Metapb.Shard::getStoreId) + .collect( + Collectors.toSet()) + .contains( + shard.getStoreId())) + .collect(Collectors.toList()); + + // add + List addShards = reqShards.stream().filter(shard -> + !curShards.stream() + .map(Metapb.Shard::getStoreId) + .collect( + Collectors.toSet()) + .contains( + shard.getStoreId())) + .collect(Collectors.toList()); + shards.addAll(addShards); + + // change leader + for (Metapb.Shard shard : shards) { + if (shard.getStoreId() == leaderStoreId) { + shard.toBuilder().setRole(Metapb.ShardRole.Leader).build(); + } else { + shard.toBuilder().setRole(Metapb.ShardRole.Follower).build(); + } + } + + } + + // @Test + public void testPriority() { + List oldPeers = new ArrayList<>(); + oldPeers.add("127.0.0.1:8001::100"); + oldPeers.add("127.0.0.1:8002::75"); + oldPeers.add("127.0.0.1:8003::50"); + + List peers = new ArrayList<>(); + peers.add("127.0.0.1:8001"); + peers.add("127.0.0.1:8003::60"); + peers.add("127.0.0.1:8004"); + + List priPeers = new ArrayList<>(); + for (String peer : peers) { + if (peer.contains("::")) { + priPeers.add(peer); + System.out.println(peer); + } else { + boolean find = false; + for (String oldPeer : oldPeers) { + if (oldPeer.contains(peer + "::")) { + find = true; + priPeers.add(oldPeer); + System.out.println(oldPeer); + break; + } + } + if (!find) { + priPeers.add(peer); + System.out.println(peer); + } + } + } + + + } + + // @Test + public void testRpcCall() throws ExecutionException, InterruptedException { + HgCmdClient client = new HgCmdClient(); + client.init(new RpcOptions(), null); + ArrayList ps = new ArrayList<>(); + Metapb.Partition p = Metapb.Partition.newBuilder() + .setGraphName("OK") + .setId(1) + // .addShards(Metapb.Shard.newBuilder().setStoreId + // (1).build()) + .build(); + ps.add(new Partition(p)); + client.createRaftNode("127.0.0.1:8511", ps, status -> { + System.out.println(status); + }).get(); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/UnitTestBase.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/UnitTestBase.java new file mode 100644 index 0000000000..86532583e8 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/UnitTestBase.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBOptions; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.store.business.BusinessHandler; +import org.apache.hugegraph.store.business.BusinessHandlerImpl; +import org.apache.hugegraph.store.meta.PartitionManager; +import org.apache.hugegraph.store.options.HgStoreEngineOptions; +import org.apache.hugegraph.store.options.RaftRocksdbOptions; +import org.apache.hugegraph.store.pd.FakePdServiceProvider; +import org.apache.hugegraph.store.pd.PdProvider; + +public class UnitTestBase { + + public final static RocksDBFactory factory = RocksDBFactory.getInstance(); + public final static String DEFAULT_TEST_TABLE = "unknown"; + private String dbPath; + + private BusinessHandler handler; + + public static boolean deleteDir(File dir) { + if (dir.isDirectory()) { + for (File file : dir.listFiles()) { + deleteDir(file); + } + } + return dir.delete(); + } + + public void initDB(String dbPath) { + this.dbPath = dbPath; + UnitTestBase.deleteDir(new File(dbPath)); + Map configMap = new HashMap<>(); + configMap.put("rocksdb.write_buffer_size", "1048576"); + configMap.put("rocksdb.bloom_filter_bits_per_key", "10"); + + RaftRocksdbOptions.initRocksdbGlobalConfig(configMap); + BusinessHandlerImpl.initRocksdb(configMap, null); + + + } + + protected BusinessHandler getBusinessHandler() { + if (handler == null) { + synchronized (this) { + if (handler == null) { + int partitionCount = 2; + HgStoreEngineOptions options = new HgStoreEngineOptions() {{ + setDataPath(dbPath); + setFakePdOptions(new HgStoreEngineOptions.FakePdOptions() {{ + setPartitionCount(partitionCount); + setPeersList("127.0.0.1"); + setStoreList("127.0.0.1"); + + }}); + }}; + + PdProvider pdProvider = new FakePdServiceProvider(options.getFakePdOptions()); + PartitionManager partitionManager = new PartitionManager(pdProvider, options); + BusinessHandler handler = new BusinessHandlerImpl(partitionManager); + } + } + } + + return handler; + } + + public RocksDBSession getDBSession(String dbName) { + RocksDBSession session = factory.queryGraphDB(dbName); + if (session == null) { + session = factory.createGraphDB(dbPath, dbName); + } + return session; + } + + public void close() { + handler.closeAll(); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/ChangeShardNumTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/ChangeShardNumTest.java new file mode 100644 index 0000000000..be0c869969 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/ChangeShardNumTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.client; + +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.util.HgStoreTestUtil; +import org.junit.Assert; +import org.junit.Test; + +/** + * 测试修改副本数 + */ +public class ChangeShardNumTest extends HgStoreClientBase { + + @Test + public void test3To1() throws PDException { + int number = 10000; + HgStoreSession session = storeClient.openSession(GRAPH_NAME); + HgStoreTestUtil.batchPut(session, TABLE_NAME, "testKey", number); + + try (HgKvIterator iterators = session.scanIterator(TABLE_NAME)) { +// Assert.assertEquals(number, HgStoreTestUtil.amountOf(iterators)); + } + + Metapb.PDConfig pdConfig = pdClient.getPDConfig(); + pdConfig = pdConfig.toBuilder().setShardCount(1).build(); + + pdClient.setPDConfig(pdConfig); + pdClient.balancePartition(); + } + + // @Test + public void test1To3() throws PDException { + int number = 10000; + HgStoreSession session = storeClient.openSession(GRAPH_NAME); + HgStoreTestUtil.batchPut(session, TABLE_NAME, "testKey", number); + + try (HgKvIterator iterators = session.scanIterator(TABLE_NAME)) { + Assert.assertEquals(number, HgStoreTestUtil.amountOf(iterators)); + } + + Metapb.PDConfig pdConfig = pdClient.getPDConfig(); + pdConfig = pdConfig.toBuilder().setShardCount(3).build(); + + pdClient.setPDConfig(pdConfig); + pdClient.balancePartition(); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/ClientTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/ClientTest.java new file mode 100644 index 0000000000..e2c9fb1c2b --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/ClientTest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.client; + +import org.junit.Assert; +import org.junit.Test; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ClientTest { + + @Test + public void testDemo() { + String s = "i am client"; + log.info("UT:{}", s); + Assert.assertTrue(s.startsWith("i")); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgAssertTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgAssertTest.java new file mode 100644 index 0000000000..ecd8bc0592 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgAssertTest.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.client; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.List; +import java.util.Map; + +import org.apache.hugegraph.store.client.util.HgAssert; +import org.junit.Test; + +public class HgAssertTest { + + @Test + public void testIsTrue1() { + try { + HgAssert.isTrue(false, "message"); + } catch (Exception e) { + assertTrue(e instanceof IllegalArgumentException); + } + } + + @Test + public void testIsTrue2() { + try { + HgAssert.isTrue(false, () -> "message"); + } catch (Exception e) { + assertTrue(e instanceof IllegalArgumentException); + } + } + + @Test + public void testIsFalse1() { + try { + HgAssert.isFalse(true, "message"); + } catch (Exception e) { + assertTrue(e instanceof IllegalArgumentException); + } + } + + @Test + public void testIsFalse2() { + try { + HgAssert.isFalse(true, () -> "message"); + } catch (Exception e) { + assertTrue(e instanceof IllegalArgumentException); + } + } + + @Test + public void testIsArgumentValid1() { + // Setup + // Run the test + HgAssert.isArgumentValid("content".getBytes(), "parameter"); + + // Verify the results + } + + @Test + public void testIsArgumentValid2() { + // Setup + // Run the test + HgAssert.isArgumentValid("str", "parameter"); + + // Verify the results + } + + @Test + public void testIsArgumentValid3() { + // Setup + // Run the test + HgAssert.isArgumentValid(List.of("value"), "parameter"); + + // Verify the results + } + + @Test + public void testIsArgumentNotNull() { + // Setup + // Run the test + HgAssert.isArgumentNotNull("obj", "parameter"); + + // Verify the results + } + + @Test + public void testIstValid() { + // Setup + // Run the test + HgAssert.istValid("content".getBytes(), "message"); + + // Verify the results + } + + @Test + public void testIsValid() { + // Setup + // Run the test + HgAssert.isValid("str", "message"); + + // Verify the results + } + + @Test + public void testIsNotNull() { + // Setup + // Run the test + HgAssert.isNotNull("obj", "message"); + + // Verify the results + } + + @Test + public void testIsContains1() { + assertTrue(HgAssert.isContains(new Object[]{"obj"}, "obj")); + } + + @Test + public void testIsInvalid1() { + assertFalse(HgAssert.isInvalid("strs")); + } + + @Test + public void testIsInvalid2() { + assertFalse(HgAssert.isInvalid("content".getBytes())); + } + + @Test + public void testIsInvalid3() { + // Setup + final Map map = Map.ofEntries(Map.entry("value", "value")); + // Verify the results + assertFalse(HgAssert.isInvalid(map)); + } + + @Test + public void testIsInvalid4() { + assertFalse(HgAssert.isInvalid(List.of("value"))); + } + + @Test + public void testIsContains2() { + assertTrue(HgAssert.isContains(List.of("item"), "item")); + } + + @Test + public void testIsNull() { + assertFalse(HgAssert.isNull("objs")); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgKvStoreTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgKvStoreTest.java new file mode 100644 index 0000000000..6cbf9a7877 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgKvStoreTest.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.client; + +import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toBytes; +import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toOwnerKey; +import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toStr; + +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgStoreSession; +import org.junit.Assert; +import org.junit.Test; + +public class HgKvStoreTest extends HgStoreClientBase { + + @Test + public void truncateTest() { + System.out.println("--- test truncateTest ---"); + HgStoreSession graph0 = storeClient.openSession("hugegraph0"); + HgStoreSession graph1 = storeClient.openSession("hugegraph1"); + + graph0.truncate(); + graph1.truncate(); + + for (int i = 0; i < 3; i++) { + HgOwnerKey key = toOwnerKey("owner-" + i, "ownerKey-" + i); + byte[] value0 = toBytes("g0 owner-" + i + ";ownerKey-" + i); + graph0.put(TABLE_NAME, key, value0); + + byte[] value1 = toBytes("g1 owner-" + i + ";ownerKey-" + i); + graph1.put(TABLE_NAME, key, value1); + } + + HgKvIterator iterator = graph0.scanIterator(TABLE_NAME); + Assert.assertTrue(iterator.hasNext()); + while (iterator.hasNext()) { + HgKvEntry entry = iterator.next(); +// log.info("key:{} value:{}", toStr(entry.key()), toStr(entry.value())); + Assert.assertEquals(0, toStr(entry.value()).indexOf("g0")); + } + iterator = graph1.scanIterator(TABLE_NAME); + Assert.assertTrue(iterator.hasNext()); + while (iterator.hasNext()) { + HgKvEntry entry = iterator.next(); +// log.info("key:{} value:{}", toStr(entry.key()), toStr(entry.value())); + Assert.assertEquals(0, toStr(entry.value()).indexOf("g1")); + } + + graph0.truncate(); + iterator = graph0.scanIterator(TABLE_NAME); + Assert.assertFalse(iterator.hasNext()); + + iterator = graph1.scanIterator(TABLE_NAME); + Assert.assertTrue(iterator.hasNext()); + while (iterator.hasNext()) { + HgKvEntry entry = iterator.next(); + // System.out.println("key:" + toStr(entry.key()) + " value:" + toStr(entry.value())); + Assert.assertEquals(0, toStr(entry.value()).indexOf("g1")); + } + } + + @Test + public void deleteTableTest() { + System.out.println("--- test deleteTableTest ---"); + HgStoreSession graph0 = storeClient.openSession("hugegraph0"); + HgStoreSession graph1 = storeClient.openSession("hugegraph1"); + graph0.truncate(); + graph1.truncate(); + + for (int i = 0; i < 2; i++) { + HgOwnerKey key = toOwnerKey("owner-" + i, "ownerKey-" + i); + byte[] value0 = toBytes("g0 owner-" + i + ";ownerKey-" + i); + graph0.put(TABLE_NAME, key, value0); + + byte[] value1 = toBytes("g1 owner-" + i + ";ownerKey-" + i); + graph1.put(TABLE_NAME, key, value1); + } + + HgKvIterator iterator = graph0.scanIterator(TABLE_NAME); + Assert.assertTrue(iterator.hasNext()); + while (iterator.hasNext()) { + HgKvEntry entry = iterator.next(); + Assert.assertEquals(0, toStr(entry.value()).indexOf("g0")); + } + iterator = graph1.scanIterator(TABLE_NAME); + Assert.assertTrue(iterator.hasNext()); + while (iterator.hasNext()) { + HgKvEntry entry = iterator.next(); + Assert.assertEquals(0, toStr(entry.value()).indexOf("g1")); + } + + graph0.deleteTable(TABLE_NAME); + Assert.assertTrue(graph0.existsTable(TABLE_NAME)); + iterator = graph0.scanIterator(TABLE_NAME); + Assert.assertFalse(iterator.hasNext()); + while (iterator.hasNext()) { + HgKvEntry entry = iterator.next(); + } + + iterator = graph1.scanIterator(TABLE_NAME); + Assert.assertTrue(iterator.hasNext()); + while (iterator.hasNext()) { + HgKvEntry entry = iterator.next(); + Assert.assertEquals(0, toStr(entry.value()).indexOf("g1")); + } + } + + @Test + public void dropTableTest() { + System.out.println("--- test dropTableTest ---"); + HgStoreSession graph0 = storeClient.openSession("hugegraph0"); + HgStoreSession graph1 = storeClient.openSession("hugegraph1"); + graph0.truncate(); + graph1.truncate(); + + for (int i = 0; i < 2; i++) { + HgOwnerKey key = toOwnerKey("owner-" + i, "ownerKey-" + i); + byte[] value0 = toBytes("g0 owner-" + i + ";ownerKey-" + i); + graph0.put(TABLE_NAME, key, value0); + + byte[] value1 = toBytes("g1 owner-" + i + ";ownerKey-" + i); + graph1.put(TABLE_NAME, key, value1); + } + + HgKvIterator iterator = graph0.scanIterator(TABLE_NAME); + Assert.assertTrue(iterator.hasNext()); + while (iterator.hasNext()) { + HgKvEntry entry = iterator.next(); + Assert.assertEquals(0, toStr(entry.value()).indexOf("g0")); + } + iterator = graph1.scanIterator(TABLE_NAME); + Assert.assertTrue(iterator.hasNext()); + while (iterator.hasNext()) { + HgKvEntry entry = iterator.next(); + Assert.assertEquals(0, toStr(entry.value()).indexOf("g1")); + } + + graph0.dropTable(TABLE_NAME); + Assert.assertTrue(graph0.existsTable(TABLE_NAME)); + iterator = graph0.scanIterator(TABLE_NAME); + Assert.assertFalse(iterator.hasNext()); + while (iterator.hasNext()) { + HgKvEntry entry = iterator.next(); + } + + iterator = graph1.scanIterator(TABLE_NAME); + Assert.assertTrue(iterator.hasNext()); + while (iterator.hasNext()) { + HgKvEntry entry = iterator.next(); + Assert.assertEquals(0, toStr(entry.value()).indexOf("g1")); + } + } + + @Test + public void moveDataPathTest() { + System.out.println("--- test moveDataPathTest ---"); + HgStoreSession graph0 = storeClient.openSession("hugegraph0"); + graph0.truncate(); + + for (int i = 0; i < 2; i++) { + HgOwnerKey key = toOwnerKey("owner-" + i, "ownerKey-" + i); + byte[] value0 = toBytes("g0 owner-" + i + ";ownerKey-" + i); + graph0.put(TABLE_NAME, key, value0); + } + + HgKvIterator iterator = graph0.scanIterator(TABLE_NAME); + Assert.assertTrue(iterator.hasNext()); + while (iterator.hasNext()) { + HgKvEntry entry = iterator.next(); + Assert.assertEquals(0, toStr(entry.value()).indexOf("g0")); + } + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgPairTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgPairTest.java new file mode 100644 index 0000000000..fd3ce3d51c --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgPairTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.client; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +import org.apache.hugegraph.store.term.Bits; +import org.apache.hugegraph.store.term.HgPair; +import org.apache.hugegraph.store.term.HgTriple; +import org.junit.Before; +import org.junit.Test; + +public class HgPairTest { + + private HgPair pair; + private HgTriple triple; + + @Before + public void setUp() { + pair = new HgPair<>("key", "value"); + triple = new HgTriple<>("x", "y", "z"); + } + + @Test + public void testPair() { + int hashCode = pair.hashCode(); + pair.toString(); + pair.setKey("key1"); + pair.setValue("value1"); + pair.getKey(); + pair.getValue(); + assertEquals(new HgPair<>("key1", "value1"), pair); + var pair2 = new HgPair<>(); + pair2.setKey("key1"); + pair2.hashCode(); + assertNotEquals(pair2, pair); + triple.getZ(); + triple.getX(); + triple.getY(); + triple.toString(); + triple.hashCode(); + triple.hashCode(); + assertEquals(triple, new HgTriple<>("x", "y", "z")); + assertNotEquals(pair2, triple); + } + + @Test + public void testBits() { + byte[] buf = new byte[4]; + Bits.putInt(buf, 0, 3); + int i = Bits.getInt(buf, 0); + assertEquals(3, i); + buf = new byte[2]; + Bits.putShort(buf, 0, 2); + int s = Bits.getShort(buf, 0); + assertEquals(2, s); + buf = new byte[4]; + Bits.put(buf, 0, new byte[]{0, 0, 0, 66}); + int toInt = Bits.toInt(buf); + assertEquals(66, toInt); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgSessionManagerRaftPDTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgSessionManagerRaftPDTest.java new file mode 100644 index 0000000000..e52ae8d2d7 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgSessionManagerRaftPDTest.java @@ -0,0 +1,685 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.client; + +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.EMPTY_BYTES; +import static org.apache.hugegraph.store.util.HgStoreTestUtil.batchPut; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.stream.Collectors; + +import org.apache.commons.io.FileUtils; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgKvStore; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgScanQuery; +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.client.grpc.KvCloseableIterator; +import org.apache.hugegraph.store.client.util.HgStoreClientConfig; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.client.util.MetricX; +import org.apache.hugegraph.store.grpc.common.ScanOrderType; +import org.apache.hugegraph.store.util.HgStoreTestUtil; +import org.junit.Assert; +import org.junit.Test; + +public class HgSessionManagerRaftPDTest extends HgStoreClientBase { + + public static final String GRAPH_NAME_X = "default/hugegraph/x"; + public static final String GRAPH_NAME_Y = "default/hugegraph/y"; + public static final String GRAPH_NAME_Z = "default/hugegraph/z"; + public static final String TABLE_NAME_1 = "table1"; + public static final String TABLE_NAME_2 = "table2"; + public static final String TABLE_NAME_3 = "table3"; + + private HgStoreSession getStoreSession() { + return storeClient.openSession(HgStoreTestUtil.GRAPH_NAME); + } + + private HgStoreSession getStoreSession(String graph) { + return storeClient.openSession(graph); + } + + @Test + public void putGet() { + System.out.println("--- test put & get ---"); + + HgStoreSession session = getStoreSession(HgStoreTestUtil.GRAPH_NAME); + + HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY"); + byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); + + long stime = System.currentTimeMillis(); + batchPut(session, TABLE_NAME, "BATCH-PUT-TEST", 30000); + System.out.println("Time is " + (System.currentTimeMillis() - stime)); + } + + public void putGet2() { + System.out.println("--- test put & get ---"); + + HgStoreSession session = getStoreSession("testGraph"); + HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY"); + byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); + long stime = System.currentTimeMillis(); + HgStoreTestUtil.batchPut(session, "testTable", "BATCH-PUT-TEST", 30000); + System.out.println("Time is " + (System.currentTimeMillis() - stime)); + } + + @Test + public void scanPartition() { + HgStoreTestUtil.println("--- test scanPartition ---"); + + HgStoreSession session = getStoreSession(); + + Iterator iterator = + session.scanIterator(TABLE_NAME, 0, 65535, HgKvStore.SCAN_HASHCODE, EMPTY_BYTES); + System.out.println(HgStoreTestUtil.amountOf(iterator)); + } + + @Test + public void check() { + System.out.println("--- test check ---"); + + HgStoreSession session = getStoreSession(); + HgKvIterator iterator = session.scanIterator(TABLE_NAME); + int count = 0; + while (iterator.hasNext()) { + iterator.next(); + count++; + } + System.out.println(count); + } + + @Test + public void putGetUnique() { + System.out.println("--- test put & get ---"); + + HgStoreSession session = getStoreSession(); + + // add timestamp into key to avoid key duplication + Date date = new Date(); + SimpleDateFormat formatter = new SimpleDateFormat("HHmmss"); + String timestamp = formatter.format(date); + + HgOwnerKey key = HgStoreTestUtil.toOwnerKey("FOR-PUT-KEY-" + timestamp); + byte[] value = HgStoreTestUtil.toBytes("FOR-PUT-VALUE"); + + Assert.assertTrue(session.put(TABLE_NAME, key, value)); + + Assert.assertEquals( + HgStoreTestUtil.toStr(value), HgStoreTestUtil.toStr(session.get(TABLE_NAME, key))); + } + + @Test + public void testBatchPutExt() throws IOException { + System.out.println("--- test batchPut ---"); + HgStoreSession session = getStoreSession(); + session.truncate(); + String keyPrefix = "BATCH-GET-UNIT"; + + Map map = HgStoreTestUtil.batchPut(session, keyPrefix, 10); + + File outputFile = new File("tmp/batch_put_list"); + if (outputFile.exists()) { + FileUtils.forceDelete(outputFile); + } + FileUtils.forceMkdir(new File("tmp/")); + + ObjectOutputStream oo = new ObjectOutputStream(new FileOutputStream(outputFile)); + oo.writeObject(map); + oo.close(); + System.out.printf("%d entries have be put into graph %s\n", map.size(), + HgStoreTestUtil.GRAPH_NAME); + + int count = 0; + HgKvIterator iterator = null; + iterator = session.scanIterator(TABLE_NAME); + while (iterator.hasNext()) { + iterator.next(); + count++; + } + Assert.assertEquals(10, count); + } + + // @Test + public void testBatchGetExt() throws IOException, ClassNotFoundException { + File outputFile = new File("tmp/batch_put_list"); + ObjectInputStream ois = new ObjectInputStream(new FileInputStream(outputFile)); + Map map = (Map) ois.readObject(); + ois.close(); + System.out.printf("%d entries get from %s\n", map.size(), outputFile.getPath()); + + HgStoreSession session = getStoreSession(); + List keyList = + map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList()); + List resList = session.batchGetOwner(TABLE_NAME, keyList); + + Assert.assertTrue( + (resList.stream().map(e -> map.containsKey(HgStoreTestUtil.toOwnerKey(e.key()))) + .allMatch(Boolean::booleanValue))); + } + + @Test + public void testBatchPutUniqueExt() throws IOException { + System.out.println("--- test batchPut ---"); + HgStoreSession session = getStoreSession(); + + // add timestamp into key to avoid key duplication + Date date = new Date(); + SimpleDateFormat formatter = new SimpleDateFormat("HHmmss"); + String timestamp = formatter.format(date); + + String keyPrefix = "BATCH-GET-UNIT-" + timestamp; + + Map map = HgStoreTestUtil.batchPut(session, keyPrefix, 30); + + File outputFile = new File("tmp/batch_put_list"); + if (outputFile.exists()) { + FileUtils.forceDelete(outputFile); + } + FileUtils.forceMkdir(new File("tmp/")); + + ObjectOutputStream oo = new ObjectOutputStream(new FileOutputStream(outputFile)); + oo.writeObject(map); + oo.close(); + System.out.printf("%d entries have be put into graph %s\n", map.size(), + HgStoreTestUtil.GRAPH_NAME); + } + + @Test + public void testBatchPutMultiGraph() throws IOException { + System.out.println("--- test testBatchPutMultiGraph ---"); + Date date = new Date(); + SimpleDateFormat formatter = new SimpleDateFormat("HHmmss"); + String timestamp = formatter.format(date); + HgStoreSession session1 = getStoreSession(GRAPH_NAME_X); + HgStoreSession session2 = getStoreSession(GRAPH_NAME_Y); + HgStoreSession session3 = getStoreSession(GRAPH_NAME_Z); + String keyPrefix1 = "BATCH-PUT-UNIT-X-" + timestamp; + String keyPrefix2 = "BATCH-PUT-UNIT-Y-" + timestamp; + String keyPrefix3 = "BATCH-PUT-UNIT-Z-" + timestamp; + HgStoreTestUtil.batchPut(session1, TABLE_NAME_1, keyPrefix1, 1000); + HgStoreTestUtil.batchPut(session1, TABLE_NAME_2, keyPrefix1, 1000); + HgStoreTestUtil.batchPut(session1, TABLE_NAME_3, keyPrefix1, 1000); + HgStoreTestUtil.batchPut(session2, TABLE_NAME_1, keyPrefix2, 1000); + HgStoreTestUtil.batchPut(session2, TABLE_NAME_2, keyPrefix2, 1000); + HgStoreTestUtil.batchPut(session2, TABLE_NAME_3, keyPrefix2, 1000); + HgStoreTestUtil.batchPut(session3, TABLE_NAME_1, keyPrefix3, 1000); + HgStoreTestUtil.batchPut(session3, TABLE_NAME_2, keyPrefix3, 1000); + HgStoreTestUtil.batchPut(session3, TABLE_NAME_3, keyPrefix3, 1000); + } + + // @Test + // CAUTION: ONLY FOR LONG！ + // 注意：目前只可以对 long 类型 value 进行 Merge 操作。 + public void merge() { + System.out.println("--- test merge (1+1=2)---"); + HgStoreSession session = getStoreSession(); + String mergeKey = "merge-key"; + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(mergeKey); + byte[] value = HgStoreTestUtil.toBytes(1L); + + System.out.println("- put " + mergeKey + ":1 -"); + session.put(TABLE_NAME, key, value); + System.out.println("- merge " + mergeKey + ":1 -"); + session.merge(TABLE_NAME, key, value); + long res = HgStoreTestUtil.toLong(session.get(TABLE_NAME, key)); + System.out.printf("after merge " + mergeKey + "=%s%n", res); + Assert.assertEquals(2L, res); + + String putStr = "19"; + session.put(TABLE_NAME, key, HgStoreTestUtil.toBytes(putStr)); + byte[] b1 = session.get(TABLE_NAME, key); + Assert.assertEquals(putStr, HgStoreTestUtil.toStr(b1)); + } + + @Test + public void delete() { + System.out.println("--- test delete ---"); + HgStoreSession session = getStoreSession(); + + String delKey = "del-key"; + String delValue = "del-value"; + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); + byte[] value = HgStoreTestUtil.toBytes(delValue); + + HgStoreTestUtil.println("- put " + delKey + ": " + delValue); + session.put(TABLE_NAME, key, value); + HgStoreTestUtil.println("- delete " + delKey); + session.delete(TABLE_NAME, key); + value = session.get(TABLE_NAME, key); + HgStoreTestUtil.println("- get " + delKey + ": " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals(EMPTY_BYTES, value); + } + + @Test + public void deleteSingle() { + System.out.println("--- test deleteSingle ---"); + HgStoreSession session = getStoreSession(); + + String delKey = "del-single-key"; + String delValue = "del-single-value"; + HgOwnerKey key = HgStoreTestUtil.toOwnerKey(delKey); + byte[] value = HgStoreTestUtil.toBytes(delValue); + + session.put(TABLE_NAME, key, value); + value = session.get(TABLE_NAME, key); + Assert.assertEquals(delValue, HgStoreTestUtil.toStr(value)); + + HgStoreTestUtil.println("- delete-single : [" + delKey + "]"); + session.deleteSingle(TABLE_NAME, key); + value = session.get(TABLE_NAME, key); + HgStoreTestUtil.println( + "- after del, get [" + delKey + "] = " + HgStoreTestUtil.toStr(value)); + Assert.assertEquals("", HgStoreTestUtil.toStr(value)); + } + + @Test + public void deleteRange() { + HgStoreTestUtil.println("--- test deleteRange ---"); + HgStoreSession session = getStoreSession(); + + String rangePrefix = "DEL-RANGE-KEY"; + String owner = "batch-delete-owner"; + Map map = + HgStoreTestUtil.batchPut( + session, + TABLE_NAME, + rangePrefix, + 10, + key -> { + return HgStoreTestUtil.toOwnerKey(owner, key); + }); + + HgOwnerKey startKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-00"); + HgOwnerKey endKey = HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-05"); + + Assert.assertTrue(session.deleteRange(TABLE_NAME, startKey, endKey)); + + HgStoreTestUtil.println( + "- after delete range from [" + HgStoreTestUtil.toStr(startKey.getKey()) + + "] to [" + + HgStoreTestUtil.toStr(endKey.getKey()) + "]"); + + for (int i = 0; i < 10; i++) { + HgOwnerKey key = + HgStoreTestUtil.toOwnerKey(owner, rangePrefix + "-" + + HgStoreTestUtil.padLeftZeros( + String.valueOf(i), 2)); + String value = HgStoreTestUtil.toStr(session.get(TABLE_NAME, key)); + + // TODO: [start,end)? + if (i < 5) { + Assert.assertEquals("", value); + } else { + // println(value); + Assert.assertNotEquals("", value); + } + } + } + + @Test + public void deletePrefix() { + System.out.println("--- test deletePrefix ---"); + HgStoreSession session = getStoreSession(); + + String prefixStr = "DEL-PREFIX-KEY"; + String owner = "batch-delete-owner"; + Map map = + HgStoreTestUtil.batchPut( + session, + TABLE_NAME, + prefixStr, + 10, + key -> { + return HgStoreTestUtil.toOwnerKey(owner, key); + }); + + // printOwner(map.entrySet().stream().map(e -> e.getKey()).collect(Collectors.toList())); + + HgOwnerKey prefix = HgStoreTestUtil.toOwnerKey(owner, prefixStr); + + Assert.assertEquals(10, HgStoreTestUtil.amountOf(session.scanIterator(TABLE_NAME, prefix))); + session.deletePrefix(TABLE_NAME, prefix); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(TABLE_NAME, prefix))); + + HgStoreTestUtil.println("- after delete by prefix:[" + prefixStr + "]"); + + for (int i = 0; i < 10; i++) { + HgOwnerKey key = + HgStoreTestUtil.toOwnerKey(owner, prefixStr + HgStoreTestUtil.toSuffix(i, 2)); + String value = HgStoreTestUtil.toStr(session.get(TABLE_NAME, key)); + Assert.assertEquals("", value); + } + } + + // @Test + // TODO: this test's result is unstable + public void scanIterator() { + HgStoreTestUtil.println("--- test scanIterator ---"); + String tableName = TABLE_NAME; + String keyName = "SCAN-ITER"; + HgStoreSession session = getStoreSession(); + HgStoreTestUtil.batchPut(session, tableName, keyName, 10000); + int count = 0; + int limit = 0; + int max = 99999; + HgKvIterator iterator = null; + + HgStoreTestUtil.println("-- test 0 element --"); + iterator = session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey("__SCAN-001"), + HgStoreTestUtil.toAllPartitionKey("__SCAN-100"), 0); + Assert.assertFalse(iterator.hasNext()); + try { + iterator.next(); + Assert.fail(); + } catch (Throwable t) { + HgStoreTestUtil.println("-- test NoSuchElementException --"); + Assert.assertTrue(t instanceof NoSuchElementException); + } + + for (int i = 1; i <= 10; i++) { + limit = i; + iterator = + session.scanIterator( + tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), limit); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + } + Assert.assertEquals(limit, count); + } + + for (int i = 1; i <= 10; i++) { + limit = i; + iterator = + session.scanIterator( + tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-00001"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-00005"), limit); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + } + if (i <= 5) { + Assert.assertEquals(limit, count); + } else { + Assert.assertEquals(5, count); + } + } + + limit = 0; + iterator = + session.scanIterator( + tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-0"), + HgStoreTestUtil.toAllPartitionKey(keyName + "-1"), limit); + + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count >= max) { + break; + } + } + Assert.assertEquals(10000, count); + + iterator = session.scanIterator(tableName); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry ignore = iterator.next(); + if (count >= max) { + break; + } + } + Assert.assertEquals(10000, count); + + iterator = + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName + "-01")); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count >= max) { + break; + } + } + Assert.assertEquals(1000, count); + } + + @Test + public void paging() { + HgStoreTestUtil.println("--- test scanIterator_range ---"); + String graph = "UNIT/paging"; + String tableName = TABLE_NAME; + String keyName = "SCAN-PAGING"; + int keyAmt = 100; + HgStoreSession session = getStoreSession(graph); + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); + int count = 0; + int limit = 0; + int max = 99999; + HgKvIterator iterator = null; + + count = 0; + + iterator = + session.scanIterator( + tableName, + HgStoreTestUtil.toAllPartitionKey(keyName + "-000"), + HgStoreClientConst.EMPTY_OWNER_KEY, + 0, + HgKvStore.SCAN_ANY, + EMPTY_BYTES); + List positionList = new LinkedList<>(); + while (iterator.hasNext()) { + HgKvEntry entry = iterator.next(); + positionList.add(iterator.position()); + if (count >= max) { + break; + } + } + + iterator = session.scanIterator(tableName, 100); + + byte[] position = positionList.get(50); + HgStoreTestUtil.println("seek: " + Arrays.toString(position)); + iterator.seek(position); + HgStoreTestUtil.println(iterator); + // Assert.assertEquals(50, HgStoreTestUtil.amountOf(iterator)); + + } + + @Test + public void truncate() { + HgStoreTestUtil.println("--- test truncate ---"); + String graph = "graph_truncate"; + String tableName = TABLE_NAME; + String keyName = "KEY_TRUNCATE"; + + HgStoreSession session = getStoreSession(graph); + HgStoreTestUtil.batchPut(session, tableName, keyName, 100); + Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + + session.truncate(); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + } + + // @Test + public void scanIteratorHuge() { + /*************** test no limit, with 10 millions **************/ + String tableName = TABLE_NAME; + String keyName = "SCAN-HUGE"; + int amount = 10_000_000; + HgStoreSession session = getStoreSession(); + + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 10)) < + 10) { + HgStoreTestUtil.batchPut(session, tableName, keyName, amount); + } + + int count = 0; + HgKvIterator iterator = session.scanIterator(tableName); + + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + } + + Assert.assertEquals(amount, count); + } + + @Test + public void scanTable() { + HgStoreSession session = getStoreSession("DEFAULT/hg1/g"); + HgStoreTestUtil.println(session.scanIterator("g+v", 10)); + } + + @Test + public void testDelGraph() { + HgStoreSession session = getStoreSession(); + session.deleteGraph(HgStoreTestUtil.GRAPH_NAME); + } + + // TODO: need figure out + // @Test + public void benchmarkScanBatch() { + HgStoreTestUtil.println("--- Benchmark scanBatch ---"); + String tableName = TABLE_NAME; + String keyName = "SCAN-BATCH"; + int keyAmt = 30001; + + HgStoreSession session = getStoreSession(); + + if (HgStoreTestUtil.amountOf( + session.scanIterator(tableName, HgStoreTestUtil.toAllPartitionKey(keyName), 1)) < + 1) { + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); + } + HgStoreTestUtil.println("-- Starting scan --"); + List> iterators = session.scanBatch(HgScanQuery.tableOf(tableName)); + Assert.assertEquals(keyAmt, HgStoreTestUtil.amountIn(iterators)); + } + + @Test + public void benchmarkScanBatch2() throws IOException { + HgStoreTestUtil.println("--- Benchmark scanBatch2 ---"); + String tableName = TABLE_NAME; + HgStoreSession session = getStoreSession(); + String keyName = "SCAN-BATCH"; + int keyAmt = 300; + Map data = + HgStoreTestUtil.batchPut(session, tableName, keyName, keyAmt); + HgStoreTestUtil.println("-- Starting scan --"); + MetricX metrics = MetricX.ofStart(); + // HgStoreTestUtil.println(session.scanIterator(tableName)); + long t = System.currentTimeMillis(); + int count = 0; + List keys = new ArrayList<>(); + data.forEach((k, v) -> keys.add(k)); + KvCloseableIterator> iterators = + session.scanBatch2( + HgScanQuery.prefixIteratorOf(tableName, keys.iterator(), + ScanOrderType.ORDER_NONE) + .builder() + .setScanType(0x40) + .build()); + while (iterators.hasNext()) { + HgKvIterator iterator = iterators.next(); + count += HgStoreTestUtil.amountOf(iterator); + } + keys.clear(); + iterators.close(); + } + + @Test + public void benchmarkScanBatchSkipDegree() throws IOException { + HgStoreTestUtil.println("--- Benchmark scanBatch2 1Owner---"); + String tableName = TABLE_NAME; + HgStoreSession session = getStoreSession(); + String keyName = "SCAN-BATCH"; + int keyAmt = 300; + byte[] owner = "Owner".getBytes(); + Map data = + HgStoreTestUtil.batchPut( + session, + tableName, + keyName, + keyAmt, + key -> { + return HgStoreTestUtil.toOwnerKey(owner, key); + }); + HgStoreTestUtil.println("-- Starting scan --"); + MetricX metrics = MetricX.ofStart(); + long t = System.currentTimeMillis(); + int count = 0; + + List keys = new ArrayList<>(); + keys.add(HgStoreTestUtil.toOwnerKey(owner, keyName)); + + KvCloseableIterator> iterators = + session.scanBatch2( + HgScanQuery.prefixIteratorOf(tableName, keys.iterator(), + ScanOrderType.ORDER_NONE) + .builder() + .setScanType(0x40) + .setSkipDegree(1) + .build()); + while (iterators.hasNext()) { + HgKvIterator iterator = iterators.next(); + count += HgStoreTestUtil.amountOf(iterator); + } + keys.clear(); + iterators.close(); + iterators = null; + + HgStoreTestUtil.println("time is " + (System.currentTimeMillis() - t)); + metrics.end(); + HgStoreTestUtil.println("*************************************************"); + HgStoreTestUtil.println("*********** Batch Scan Completed **************"); + HgStoreTestUtil.println("Total : " + (metrics.past() / 1000) + " (sec)"); + HgStoreTestUtil.println(" Sum : " + (MetricX.getIteratorWait() / 1000) + " (sec)"); + HgStoreTestUtil.println(" Amt : " + MetricX.getIteratorCount() + " (scans)."); + HgStoreTestUtil.println(" Avg : " + MetricX.getIteratorWaitAvg() + " (ms)"); + HgStoreTestUtil.println(" Max : " + MetricX.getIteratorWaitMax() + " (ms)"); + HgStoreTestUtil.println(" Fail : " + metrics.getFailureCount() + " (times)"); + HgStoreTestUtil.println( + " Page : " + HgStoreClientConfig.of().getNetKvScannerPageSize() + " (KVs)"); + HgStoreTestUtil.println(" size is " + count); + HgStoreTestUtil.println("*************************************************"); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgStoreClientBase.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgStoreClientBase.java new file mode 100644 index 0000000000..18d393f9c9 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgStoreClientBase.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.client; + +import org.apache.hugegraph.pd.client.PDClient; +import org.apache.hugegraph.pd.client.PDConfig; +import org.apache.hugegraph.store.HgStoreClient; +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.UnitTestBase; +import org.junit.After; +import org.junit.Before; + +public class HgStoreClientBase { + + protected static String GRAPH_NAME = "testGraphName"; + protected static String TABLE_NAME = UnitTestBase.DEFAULT_TEST_TABLE; + private static final String PD_ADDRESS = "127.0.0.1:8686"; + protected HgStoreClient storeClient; + protected PDClient pdClient; + + @Before + public void setup() throws Exception { + storeClient = HgStoreClient.create(PDConfig.of(PD_ADDRESS) + .setEnableCache(true)); + pdClient = storeClient.getPdClient(); + + HgStoreSession session = storeClient.openSession(TABLE_NAME); + session.dropTable(TABLE_NAME); + session.truncate(); + } + + @After + public void teardown() { + // pass + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgStoreClientTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgStoreClientTest.java new file mode 100644 index 0000000000..82bc2f7eae --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgStoreClientTest.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.client; + +import static org.apache.hugegraph.store.client.util.HgStoreClientConst.ALL_PARTITION_OWNER; + +import java.util.List; + +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgKvStore; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.util.HgStoreTestUtil; +import org.junit.Assert; +import org.junit.Test; + +public class HgStoreClientTest extends HgStoreClientBase { + + public static final byte[] EMPTY_BYTES = new byte[0]; + + @Test + public void testPutData() { + HgStoreSession session = storeClient.openSession(GRAPH_NAME); + long start = System.currentTimeMillis(); + int loop = 100000; + session.truncate(); + HgStoreTestUtil.batchPut(session, TABLE_NAME, "testKey", loop); + + System.out.println("Time is " + (System.currentTimeMillis() - start)); + try (HgKvIterator iterator = session.scanIterator(TABLE_NAME)) { + Assert.assertEquals(loop, HgStoreTestUtil.amountOf(iterator)); + } + } + + @Test + public void testPutData2() { + String GRAPH_NAME = "testGRAPH_NAME2"; + HgStoreSession session = storeClient.openSession(GRAPH_NAME); + long start = System.currentTimeMillis(); + int loop = 100000; + session.truncate(); + HgStoreTestUtil.batchPut(session, TABLE_NAME, "testKey", loop); + + System.out.println("Time is " + (System.currentTimeMillis() - start)); + try (HgKvIterator iterator = session.scanIterator(TABLE_NAME)) { + Assert.assertEquals(loop, HgStoreTestUtil.amountOf(iterator)); + } + } + + @Test + public void testScan() throws PDException { + + HgStoreSession session = storeClient.openSession(GRAPH_NAME); + HgStoreTestUtil.batchPut(session, TABLE_NAME, "testKey", 12); + + int count = 0; + byte[] position = null; + try (HgKvIterator iterator = session.scanIterator(TABLE_NAME)) { + while (iterator.hasNext()) { + iterator.next(); + position = iterator.position(); + dumpPosition(position); + if (++count > 5) { + break; + } + } + } + + System.out.println("--------------------------------"); + try (HgKvIterator iterator = session.scanIterator(TABLE_NAME)) { + iterator.seek(position); + while (iterator.hasNext()) { + iterator.next(); + dumpPosition(iterator.position()); + } + } + + System.out.println("--------------------------------"); + + byte[] start = new byte[]{0x0}; + byte[] end = new byte[]{-1}; + try (HgKvIterator iterator = session.scanIterator(TABLE_NAME, + HgOwnerKey.of( + ALL_PARTITION_OWNER, + start), + HgOwnerKey.of( + ALL_PARTITION_OWNER, + end))) { + iterator.seek(position); + while (iterator.hasNext()) { + iterator.next(); + dumpPosition(iterator.position()); + } + } + } + + public void dumpPosition(byte[] b) { + byte[] buf = new byte[Long.BYTES]; + System.arraycopy(b, 0, buf, 0, Long.BYTES); + // long storeId = HgStoreTestUtil.toLong(buf); + buf = new byte[Integer.BYTES]; + System.arraycopy(b, Long.BYTES, buf, 0, Integer.BYTES); + // int partId = HgStoreTestUtil.toInt(buf); + // String key = new String(b); + + // System.out.println(" " + storeId + ", " + partId + ", " + key); + } + + // @Test + // TODO: unstable + public void testDeleteData() { + HgStoreSession session = storeClient.openSession(GRAPH_NAME); + int ownerCode = 1; + HgStoreTestUtil.batchPut(session, TABLE_NAME, "T", 10, (key) -> { + return HgStoreTestUtil.toOwnerKey(ownerCode, key); + } + ); + try (HgKvIterator iterators = session.scanIterator(TABLE_NAME)) { +// while (iterators.hasNext()){ +// System.out.println(new String(iterators.next().key())); +// } + Assert.assertEquals(10, HgStoreTestUtil.amountOf(iterators)); + } + session.beginTx(); + session.deletePrefix(TABLE_NAME, HgStoreTestUtil.toOwnerKey(ownerCode, "T")); + session.commit(); + + System.out.println("================================="); + try (HgKvIterator iterators = session.scanIterator(TABLE_NAME)) { + Assert.assertEquals(0, HgStoreTestUtil.amountOf(iterators)); +// while (iterators.hasNext()){ +// System.out.println(new String(iterators.next().key())); +// } + } + } + + // @Test + // TODO: unstable + public void testDropTable() throws PDException { + HgStoreSession session = storeClient.openSession(GRAPH_NAME); + + String table1 = TABLE_NAME; + session.createTable(table1); + HgStoreTestUtil.batchPut(session, table1, "testKey", 1000); + + try (HgKvIterator iterators = session.scanIterator(table1)) { + Assert.assertEquals(1000, HgStoreTestUtil.amountOf(iterators)); + } + + session.dropTable(table1); + try (HgKvIterator iterators = session.scanIterator(table1)) { + Assert.assertEquals(0, HgStoreTestUtil.amountOf(iterators)); + } + + deleteGraph(GRAPH_NAME); + } + + public void deleteGraph(String GRAPH_NAME) throws PDException { + HgStoreSession session = storeClient.openSession(GRAPH_NAME); + session.deleteGraph(GRAPH_NAME); + pdClient.delGraph(GRAPH_NAME); + + Metapb.Graph graph = null; + try { + graph = pdClient.getGraph(GRAPH_NAME); + } catch (PDException e) { + Assert.assertEquals(103, e.getErrorCode()); + } + Assert.assertNull(graph); + } + + @Test + public void testScanPartition() throws PDException { + // testPutData(); + List partitions = pdClient.getPartitions(0, "DEFAULT/hugegraph/g"); + HgStoreSession session = storeClient.openSession("DEFAULT/hugegraph/g"); + for (Metapb.Partition partition : partitions) { + try (HgKvIterator iterators = session.scanIterator("g+v", + (int) (partition.getStartKey()), + (int) (partition.getEndKey()), + HgKvStore.SCAN_HASHCODE, + EMPTY_BYTES)) { + + System.out.println( + " " + partition.getId() + " " + HgStoreTestUtil.amountOf(iterators)); + } + } + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeSessionTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeSessionTest.java new file mode 100644 index 0000000000..25151e4c84 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeSessionTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.client; + +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.util.HgStoreTestUtil; +import org.junit.Assert; + +/** + * created on 2021/10/12 + */ +public class HgStoreNodeSessionTest { + + private static final HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance(); + private static HgStoreNode node; + + // @BeforeClass + public static void init() { + node = nodeManager.addNode(HgStoreTestUtil.GRAPH_NAME, + nodeManager.getNodeBuilder().setAddress("localhost:9080") + .build()); + } + + private static HgStoreSession getStoreSession() { + return node.openSession(HgStoreTestUtil.GRAPH_NAME); + } + + private HgStoreNode getOneNode() { + return node; + } + + // @Test + public void truncate() { + HgStoreTestUtil.println("--- test truncate ---"); + String tableName = "UNIT_TRUNCATE_1"; + String keyName = "KEY_TRUNCATE"; + + HgStoreSession session = getStoreSession(); + HgStoreTestUtil.batchPut(session, tableName, keyName, 100); + Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + + String tableName2 = "UNIT_TRUNCATE_2"; + HgStoreTestUtil.batchPut(session, tableName2, keyName, 100); + Assert.assertEquals(100, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); + + session.truncate(); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName))); + Assert.assertEquals(0, HgStoreTestUtil.amountOf(session.scanIterator(tableName2))); + } + +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeStateTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeStateTest.java new file mode 100644 index 0000000000..f1bb94c333 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeStateTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.client; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.hugegraph.store.util.HgStoreTestUtil; +import org.junit.Test; + +public class HgStoreNodeStateTest { + private static final HgStoreNodeManager NODE_MANAGER = HgStoreNodeManager.getInstance(); + static int nodeNumber = 0; + + static { + registerNode(HgStoreTestUtil.GRAPH_NAME, Long.valueOf(nodeNumber++), "localhost:9180"); + registerNode(HgStoreTestUtil.GRAPH_NAME, Long.valueOf(nodeNumber++), "localhost:9280"); + registerNode(HgStoreTestUtil.GRAPH_NAME, Long.valueOf(nodeNumber++), "localhost:9380"); + } + + private static void registerNode(String graphName, Long nodeId, String address) { + NODE_MANAGER.addNode(graphName, + NODE_MANAGER.getNodeBuilder().setNodeId(nodeId).setAddress(address) + .build()); + } + + + @Test + public void isNodeHealthy() { + AtomicInteger count = new AtomicInteger(0); + + for (int i = 0; i < 100; i++) { + NODE_MANAGER.getStoreNodes(HgStoreTestUtil.GRAPH_NAME) + .stream().map( + node -> { + System.out.println(node.getNodeId() + " " + node.getAddress() + + "is healthy: " + node.isHealthy()); + return node.isHealthy(); + } + ).count(); + + Thread.yield(); + } + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeStreamTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeStreamTest.java new file mode 100644 index 0000000000..aeee3da6b7 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/HgStoreNodeStreamTest.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.client; + +import static org.apache.hugegraph.store.util.HgStoreTestUtil.GRAPH_NAME; +import static org.apache.hugegraph.store.util.HgStoreTestUtil.batchPut; +import static org.apache.hugegraph.store.util.HgStoreTestUtil.println; +import static org.apache.hugegraph.store.util.HgStoreTestUtil.toOwnerKey; + +import java.util.NoSuchElementException; + +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.client.HgStoreNode; +import org.apache.hugegraph.store.client.HgStoreNodeManager; +import org.apache.hugegraph.store.util.HgStoreTestUtil; +import org.junit.Assert; + +/** + * created on 2021/10/12 + */ +public class HgStoreNodeStreamTest { + private static final HgStoreNodeManager nodeManager = HgStoreNodeManager.getInstance(); + private static HgStoreNode node; + + // @BeforeClass + public static void init() { + node = nodeManager.addNode(HgStoreTestUtil.GRAPH_NAME, + nodeManager.getNodeBuilder().setAddress("localhost:9080") + .build()); + } + + private static HgStoreSession getStoreSession() { + return node.openSession(HgStoreTestUtil.GRAPH_NAME); + } + + private HgStoreNode getOneNode() { + return node; + } + + // @Test + public void scanIterator() { + + HgStoreTestUtil.println("--- test scanIterator ---"); + String tableName = "UNIT_SCAN"; + String keyName = "SCAN-ITER"; + HgStoreSession session = getStoreSession(); + HgStoreTestUtil.batchPut(session, tableName, keyName, 10000); + int count = 0; + int limit = 0; + int max = 99999; + HgKvIterator iterator = null; + + HgStoreTestUtil.println("-- test 0 element --"); + iterator = + session.scanIterator(tableName, HgStoreTestUtil.toOwnerKey("__SCAN-001"), HgStoreTestUtil.toOwnerKey("__SCAN-100"), + 0); + Assert.assertFalse(iterator.hasNext()); + try { + iterator.next(); + Assert.fail(); + } catch (Throwable t) { + HgStoreTestUtil.println("-- test NoSuchElementException --"); + Assert.assertTrue(t instanceof NoSuchElementException); + } + + HgStoreTestUtil.println("-- test limit 1 to 10 --"); + for (int i = 1; i <= 10; i++) { + HgStoreTestUtil.println("- limit " + i + " -"); + limit = i; + iterator = session.scanIterator(tableName, HgStoreTestUtil.toOwnerKey(keyName + "-0"), + HgStoreTestUtil.toOwnerKey(keyName + "-1"), limit); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + HgStoreTestUtil.println(entry); + } + Assert.assertEquals(limit, count); + } + + HgStoreTestUtil.println("-- test limit 1 to 10 not enough --"); + for (int i = 1; i <= 10; i++) { + HgStoreTestUtil.println("- limit " + i + " -"); + limit = i; + iterator = session.scanIterator(tableName, + HgStoreTestUtil.toOwnerKey(keyName + "-00001"), + HgStoreTestUtil.toOwnerKey(keyName + "-00005"), limit); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + HgStoreTestUtil.println(entry); + } + if (i <= 5) { + Assert.assertEquals(limit, count); + } else { + Assert.assertEquals(5, count); + } + + } + + HgStoreTestUtil.println("-- test limit 0 (no limit) --"); + limit = 0; + iterator = session.scanIterator(tableName, HgStoreTestUtil.toOwnerKey(keyName + "-0"), + HgStoreTestUtil.toOwnerKey(keyName + "-1"), limit); + + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 1000 == 0) { + HgStoreTestUtil.println(entry); + } + if (count >= max) break; + } + Assert.assertEquals(10000, count); + + HgStoreTestUtil.println("-- test scan all --"); + iterator = session.scanIterator(tableName); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 100 == 0) { + HgStoreTestUtil.println(entry); + } + if (count >= max) break; + } + Assert.assertEquals(10000, count); + + HgStoreTestUtil.println("-- test scan prefix --"); + iterator = session.scanIterator(tableName, HgStoreTestUtil.toOwnerKey(keyName + "-01")); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 100 == 0) { + HgStoreTestUtil.println(entry); + } + if (count >= max) break; + } + Assert.assertEquals(1000, count); + + } + + //// @Test + public void scanIteratorBenchmark() { + /*************** test no limit, with 10 millions **************/ + String tableName = "UNIT_HUGE"; + String keyName = "SCAN-HUGE"; + HgStoreSession session = getStoreSession(); + //batchPut(session,tableName, keyName, 10000000); + int count = 0; + int limit = 0; + HgKvIterator iterator = null; + + limit = 0; + iterator = session.scanIterator(tableName); + count = 0; + while (iterator.hasNext()) { + count++; + HgKvEntry entry = iterator.next(); + if (count % 100000 == 0) { + HgStoreTestUtil.println(entry); + } + } + + Assert.assertEquals(10000000, count); + + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/graph/GraphStoreClientTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/graph/GraphStoreClientTest.java new file mode 100644 index 0000000000..4514ef676e --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/client/graph/GraphStoreClientTest.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.client.graph; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.hugegraph.store.grpc.GraphStoreGrpc; +import org.apache.hugegraph.store.grpc.GraphStoreGrpc.GraphStoreStub; +import org.apache.hugegraph.store.grpc.Graphpb.ScanPartitionRequest; +import org.apache.hugegraph.store.grpc.Graphpb.ScanPartitionRequest.Reply; +import org.apache.hugegraph.store.grpc.Graphpb.ScanResponse; +import org.junit.Test; + +import com.google.protobuf.util.JsonFormat; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class GraphStoreClientTest { + + private final AtomicInteger sum = new AtomicInteger(); + ConcurrentHashMap> observers = + new ConcurrentHashMap<>(); + + @Test + public void getData() { + long start = System.currentTimeMillis(); + String[] addresses = new String[]{"10.14.139.71:8500", + "10.14.139.70:8500", + "10.14.139.69:8500"}; + Arrays.stream(addresses).parallel().forEach(address -> { + int size = 72; + CountDownLatch latch = new CountDownLatch(size); + + for (int i = 0; i < size; i++) { + int finalI = i; + new Thread(() -> getData(finalI, latch, address)).start(); + } + try { + latch.await(); + + } catch (InterruptedException e) { + e.printStackTrace(); + } + }); + long end = System.currentTimeMillis(); + System.out.println( + "all rows are: " + sum.get() + ", end: " + (end - start)); + } + + public void getData(int pId, CountDownLatch latch, String address) { + try { + ScanPartitionRequest.Builder builder = + ScanPartitionRequest.newBuilder(); + ScanPartitionRequest.Request.Builder srb = + ScanPartitionRequest.Request.newBuilder(); + ArrayList properties = new ArrayList<>() {{ + add(2L); + add(3L); + }}; + ScanPartitionRequest.Request request = + srb.setGraphName("DEFAULT/hugegraph0/g") + .setScanType( + ScanPartitionRequest.ScanType.SCAN_VERTEX).setBoundary(0x10) + // .addAllProperties(properties) + // .setCondition("element.id().asString().equals + // ('1:marko1')") + .setPartitionId(pId).build(); + ManagedChannel c = + ManagedChannelBuilder.forTarget(address) + .usePlaintext().build(); + int maxSize = 1024 * 1024 * 1024; + GraphStoreStub stub; + stub = GraphStoreGrpc.newStub(c) + .withMaxInboundMessageSize(maxSize) + .withMaxOutboundMessageSize(maxSize); + + AtomicInteger count = new AtomicInteger(); + long start = System.currentTimeMillis(); + long id = Thread.currentThread().getId(); + StreamObserver ro = + new StreamObserver() { + @Override + public void onNext(ScanResponse value) { + try { + int edgeSize = value.getEdgeCount(); + int vertexSize = value.getVertexCount(); + if (request.getScanType().equals( + ScanPartitionRequest.ScanType.SCAN_VERTEX)) { + count.getAndAdd(vertexSize); + } else { + count.getAndAdd(edgeSize); + } + String print = JsonFormat.printer().print + (value); + System.out.println(print); + ScanPartitionRequest.Builder builder = + ScanPartitionRequest.newBuilder(); + builder.setScanRequest(request); + Reply.Builder reply = Reply.newBuilder(); + reply.setSeqNo(1); + builder.setReplyRequest(reply); + observers.get(id).onNext(builder.build()); + + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Override + public void onError(Throwable t) { + System.out.println(t); + latch.countDown(); + } + + @Override + public void onCompleted() { + long time = System.currentTimeMillis() - start; + System.out.println( + "scan id :" + pId + ", complete: " + + count.get() + ",time: " + time); + sum.addAndGet(count.get()); + latch.countDown(); + } + }; + StreamObserver observer = + stub.scanPartition(ro); + observers.put(id, observer); + builder.setScanRequest(request); + observer.onNext(builder.build()); + } catch (Exception e) { + e.printStackTrace(); + } + } + + + @Test + public void getDataSingle() { + CountDownLatch latch = new CountDownLatch(1); + new Thread(() -> getData(58, latch, "10.14.139.71:8500")).start(); + try { + latch.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + System.out.println("all rows are: " + sum.get()); + } + + @Test + public void getNativeDataSingle() { + CountDownLatch latch = new CountDownLatch(1); + new Thread(() -> getData(0, latch, "127.0.0.1:8500")).start(); + try { + latch.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + System.out.println("all rows are: " + sum.get()); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/common/BaseCommonTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/common/BaseCommonTest.java index 0067b11ae7..7508ec4d5e 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/common/BaseCommonTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/common/BaseCommonTest.java @@ -21,7 +21,6 @@ import org.junit.BeforeClass; public class BaseCommonTest { - @BeforeClass public static void beforeClass() throws Exception { } diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/common/BitsTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/common/BitsTest.java index afa1f2baba..a665b4d764 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/common/BitsTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/common/BitsTest.java @@ -22,7 +22,6 @@ // import org.junit.Test; public class BitsTest { - // @Test public void test() { for (int i = 0; i < Integer.MAX_VALUE; i = i + 10) { diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/common/ByteBufferAllocatorTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/common/ByteBufferAllocatorTest.java index 0faf3457c7..90aae3a2e8 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/common/ByteBufferAllocatorTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/common/ByteBufferAllocatorTest.java @@ -27,7 +27,6 @@ @Slf4j public class ByteBufferAllocatorTest extends BaseCommonTest { - @Test public void getAndReleaseTest() throws InterruptedException { final CountDownLatch latch = new CountDownLatch(2); diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/common/CommonSuiteTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/common/CommonSuiteTest.java index 092cc804fe..2ef65cf3b0 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/common/CommonSuiteTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/common/CommonSuiteTest.java @@ -30,5 +30,4 @@ @Slf4j public class CommonSuiteTest { - } diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/BaseCoreTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/BaseCoreTest.java new file mode 100644 index 0000000000..3ecbb1da73 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/BaseCoreTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core; + +import java.io.File; +import java.io.IOException; + +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.BeforeClass; + + +public class BaseCoreTest { + + @BeforeClass + public static void setup() throws Exception { + + } + + public static void deleteDirectory(File dir) { + try { + FileUtils.deleteDirectory(dir); + } catch (IOException e) { + System.out.printf("Failed to start ....,%s%n", e.getMessage()); + } + } + + @After + public void teardown() throws Exception { + // pass + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreSuiteTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreSuiteTest.java new file mode 100644 index 0000000000..9781b06a7b --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreSuiteTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core; + +import lombok.extern.slf4j.Slf4j; + +// TODO: uncomment it until all test can run free. +//@RunWith(Suite.class) +//@Suite.SuiteClasses({ +// HgCmdClientTest.class, +// HgSnapshotHandlerTest.class, +// RaftUtilsTest.class, +// RaftOperationTest.class, +// UnsafeUtilTest.class, +// CopyOnWriteCacheTest.class, +// MetricServiceTest.class, +// TaskManagerTest.class, +// CleanTaskTest.class, +// MetadataKeyHelperTest.class, +// HgStoreEngineTest.class, +// PartitionEngineTest.class, +// PartitionManagerTest.class, +// PartitionMetaStoreWrapperTest.class, +// ZipUtilsTest.class, +// MiscUtilClassTest.class, +// PartitionInstructionProcessorTest.class, +// // 尽量放到最后 +// HgBusinessImplTest.class +//}) + +@Slf4j +public class CoreSuiteTest { + +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreTest.java new file mode 100644 index 0000000000..6af95dac56 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreTest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core; + +import org.junit.Assert; +import org.junit.Test; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class CoreTest { + + @Test + public void testDemo() { + String s = "i am core"; + log.info("UT:{}", s); + Assert.assertTrue(s.startsWith("i")); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/HgBusinessImplTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/HgBusinessImplTest.java new file mode 100644 index 0000000000..d30e37e8a3 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/HgBusinessImplTest.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.store.UnitTestBase; +import org.apache.hugegraph.store.business.BusinessHandler; +import org.apache.hugegraph.util.Bytes; +import org.junit.Assert; +import org.junit.Test; + +import lombok.extern.slf4j.Slf4j; + + +@Slf4j +public class HgBusinessImplTest extends StoreEngineTestBase { + + public static final String TABLE_NAME = UnitTestBase.DEFAULT_TEST_TABLE; + public BusinessHandler getBusinessHandler() { + return getStoreEngine().getBusinessHandler(); + } + + @Test + public void testPut() throws InterruptedException { + + String graph1 = "testPut-graph1"; + String graph2 = "testPut-graph2"; + + createPartitionEngine(0, graph1); + createPartitionEngine(0, graph2); + + BusinessHandler handler = getBusinessHandler(); + handler.doPut(graph1, 0, TABLE_NAME, "key1".getBytes(), "value1".getBytes()); + handler.doPut(graph1, 0xF, TABLE_NAME, "key2".getBytes(), "value1".getBytes()); + handler.doPut(graph1, 0xFF, TABLE_NAME, "key3".getBytes(), "value1".getBytes()); + handler.doPut(graph1, 0xFFF, TABLE_NAME, "key4".getBytes(), "value1".getBytes()); + + handler.doPut(graph2, 0, TABLE_NAME, "key21".getBytes(), "value1".getBytes()); + handler.doPut(graph2, 0xF, TABLE_NAME, "key22".getBytes(), "value1".getBytes()); + handler.doPut(graph2, 0xFF, TABLE_NAME, "key23".getBytes(), "value1".getBytes()); + handler.doPut(graph2, 0xFFF, TABLE_NAME, "key24".getBytes(), "value1".getBytes()); + + System.out.println("--------------------dump all -------"); + dump(handler, graph1, 0); + System.out.println("--------------------dump scan 0 0xff -------"); + ScanIterator iterator = handler.scan(graph1, TABLE_NAME, 0, 0xff); + int count = 0; + while (iterator.hasNext()) { + RocksDBSession.BackendColumn entry = iterator.next(); + System.out.println(new String(entry.name) + " -- " + Bytes.toHex(entry.name)); + count++; + } + + Assert.assertEquals(2, count); + + System.out.println("--------------------dump scan prefix -------"); + iterator = handler.scanPrefix(graph1, 0, TABLE_NAME, "key".getBytes()); + + count = 0; + while (iterator.hasNext()) { + RocksDBSession.BackendColumn entry = iterator.next(); + System.out.println(new String(entry.name) + " -- " + Bytes.toHex(entry.name)); + count++; + } + + Assert.assertEquals(4, count); + System.out.println("--------------------dump scan range key1 key4 -------"); + iterator = handler.scan(graph1, 0, TABLE_NAME, "key1".getBytes(), "key4".getBytes(), + ScanIterator.Trait.SCAN_LT_END); + count = 0; + while (iterator.hasNext()) { + RocksDBSession.BackendColumn entry = iterator.next(); + System.out.println(new String(entry.name) + " -- " + Bytes.toHex(entry.name)); + count++; + } + + Assert.assertEquals(3, count); + boolean cleanResult = handler.cleanPartition(graph1, 0); + Assert.assertTrue(cleanResult); + Thread.sleep(2000); + + handler.closeAll(); + } + + @Test + public void testLoadSnapshot() throws InterruptedException { + + String graph1 = "LoadSnapshot-graph1"; + createPartitionEngine(0, graph1); + BusinessHandler handler = getBusinessHandler(); + + for (int i = 0; i < 256; i++) { + handler.doPut(graph1, i, TABLE_NAME, ("key" + i).getBytes(), "value1".getBytes()); + } + ScanIterator iterator = handler.scanAll(graph1, TABLE_NAME); + + System.out.println(iterator.count()); + + String snapshotPath; + try (RocksDBSession session = handler.getSession(0)) { + snapshotPath = session.getDbPath(); + } + + handler.closeAll(); + + System.out.println("start loadSnapshot"); + handler.loadSnapshot(snapshotPath, graph1, 0, 10); + iterator = handler.scanAll(graph1, TABLE_NAME); + Assert.assertEquals(255, iterator.count()); + try (RocksDBSession session = handler.getSession(0)) { + System.out.println(session.getDbPath()); + } + + CountDownLatch latch = new CountDownLatch(1); + RocksDBFactory.getInstance() + .addRocksdbChangedListener(new RocksDBFactory.RocksdbChangedListener() { + @Override + public void onCompacted(String dbName) { + RocksDBFactory.RocksdbChangedListener.super.onCompacted(dbName); + } + + @Override + public void onDBDeleteBegin(String dbName, String filePath) { + RocksDBFactory.RocksdbChangedListener.super.onDBDeleteBegin(dbName, + filePath); + } + + @Override + public void onDBDeleted(String dbName, String filePath) { + latch.countDown(); + } + + @Override + public void onDBSessionReleased(RocksDBSession dbSession) { + RocksDBFactory.RocksdbChangedListener.super.onDBSessionReleased( + dbSession); + } + }); + + latch.await(1, TimeUnit.SECONDS); + + } + + public void dump(BusinessHandler handler, String graph, int partId) { + ScanIterator cfIterator = handler.scanRaw(graph, partId, 0); + while (cfIterator.hasNext()) { + try (ScanIterator iterator = cfIterator.next()) { + byte[] cfName = cfIterator.position(); + System.out.println(graph + "-" + partId + "-" + new String(cfName) + "--------"); + while (iterator.hasNext()) { + RocksDBSession.BackendColumn col = iterator.next(); + System.out.println(new String(col.name) + " -- " + Bytes.toHex(col.name)); + } + } + } + cfIterator.close(); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/HgCmdClientTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/HgCmdClientTest.java new file mode 100644 index 0000000000..d31e01724a --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/HgCmdClientTest.java @@ -0,0 +1,359 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core; + +import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toIntBytes; +import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toOwnerKey; +import static org.apache.hugegraph.store.client.util.HgStoreClientUtil.toStr; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.hugegraph.pd.client.PDClient; +import org.apache.hugegraph.pd.client.PDConfig; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgStoreClient; +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.UnitTestBase; +import org.apache.hugegraph.store.cmd.BatchPutRequest; +import org.apache.hugegraph.store.cmd.BatchPutResponse; +import org.apache.hugegraph.store.cmd.CleanDataRequest; +import org.apache.hugegraph.store.cmd.CleanDataResponse; +import org.apache.hugegraph.store.cmd.HgCmdClient; +import org.apache.hugegraph.store.meta.Store; +import org.apache.hugegraph.store.pd.DefaultPdProvider; +import org.apache.hugegraph.store.pd.PdProvider; +import org.junit.Assert; +import org.junit.Test; + +import com.alipay.sofa.jraft.JRaftUtils; +import com.alipay.sofa.jraft.option.RpcOptions; +import com.alipay.sofa.jraft.util.Endpoint; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class HgCmdClientTest { + + private static AtomicLong id; + private final String pdAddress = "127.0.0.1:8686"; + private final String graphName = "hugegraph"; + private final String tableName = UnitTestBase.DEFAULT_TEST_TABLE; + private PdProvider pdProvider; + private HgCmdClient hgCmdClient; + private HgStoreClient storeClient; + private PDClient pdClient; + + public static String getMd5(String txt) { + String rs = ""; + String[] hexDigits = + {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"}; + try { + MessageDigest messageDigest = MessageDigest.getInstance("MD5"); + byte[] b = messageDigest.digest(txt.getBytes()); + StringBuffer resultSb = new StringBuffer(); + for (int i = 0; i < b.length; i++) { + int n = b[i]; + if (n < 0) { + n = 256 + n; + } + int d1 = n / 16; + int d2 = n % 16; + resultSb.append(hexDigits[d1] + hexDigits[d2]); + } + rs = resultSb.toString(); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + } + return rs; + } + + public static Long getId() { + // 如果需要更长 或者更大冗余空间，只需要 time * 10^n 即可 + // 当前可保证 1 毫秒 生成 10000 条不重复 + Long time = Long.valueOf(new SimpleDateFormat("HHmmssSSS").format(new Date())) * 10000 + + (long) (Math.random() * 100); +// Long time = Long.valueOf(new SimpleDateFormat("MMddhhmmssSSS").format(new Date()) +// .toString()); +// System.out.println(time); + if (id == null) { + id = new AtomicLong(time); + return id.get(); + } + if (time <= id.get()) { + id.addAndGet(1); + } else { + id = new AtomicLong(time); + } + return id.get(); + } + + @Test + public void testGetStoreInfo() { + + hgCmdClient = new HgCmdClient(); + pdProvider = new DefaultPdProvider(pdAddress); + hgCmdClient.init(new RpcOptions(), null); + + Store response = hgCmdClient.getStoreInfo(pdAddress); + + } + + // @Test + public void testBatchPut() { + + hgCmdClient = new HgCmdClient(); + pdProvider = new DefaultPdProvider(pdAddress); + hgCmdClient.init(new RpcOptions(), new HgCmdClient.PartitionAgent() { + @Override + public Endpoint getPartitionLeader(String graphName, int partitionId) { + Metapb.Shard shard = pdProvider.getPartitionLeader(graphName, partitionId); + return JRaftUtils.getEndPoint( + pdProvider.getStoreByID(shard.getStoreId()).getRaftAddress()); + } + }); + + storeClient = HgStoreClient.create(PDConfig.of(pdAddress) + .setEnableCache(true)); + HgStoreSession session = storeClient.openSession(graphName); + pdClient = storeClient.getPdClient(); + session.createTable(tableName); + String createGraph = "create_graph"; + HgOwnerKey hgOwnerKey = toOwnerKey(createGraph); + // 需要写数据，才会创建图 + session.put(tableName, + hgOwnerKey, createGraph.getBytes(StandardCharsets.UTF_8)); + Assert.assertEquals(createGraph, toStr(session.get(tableName, hgOwnerKey))); + + Integer partId = 0; + String key = "key-1"; + List kvs = new LinkedList<>(); + int x = 0; + for (int i = 1; i <= 3; i++) { + key = "key-" + i; + BatchPutRequest.KV kv = BatchPutRequest.KV.of(tableName, 1, + key.getBytes(StandardCharsets.UTF_8), + key.getBytes(StandardCharsets.UTF_8)); + kvs.add(kv); + + BatchPutRequest request = new BatchPutRequest(); + request.setGraphName(graphName); + request.setPartitionId(partId); + request.setEntries(kvs); + + try { + BatchPutResponse response = hgCmdClient.batchPut(request); + if (response == null) { + log.error("response is null "); + } else if (response.getStatus() == null) { + log.error("response status is null"); + } + + log.info("response status:{} {}", response.getStatus(), i); + + Assert.assertTrue(response.getStatus().isOK()); + } catch (Exception e) { + e.printStackTrace(); + } + } + + HgKvIterator hgKvIterator = session.scanIterator(tableName); + Assert.assertTrue(hgKvIterator.hasNext()); + boolean findKey = false; + while (hgKvIterator.hasNext()) { + HgKvEntry entry = hgKvIterator.next(); + if (toStr(entry.key()).equals(key) && toStr(entry.value()).equals(key)) { + log.info("key={} value={}", toStr(entry.key()), toStr(entry.value())); + findKey = true; + } + } + Assert.assertTrue(findKey); + } + + // @Test + public void testCleanData() { + + hgCmdClient = new HgCmdClient(); + pdProvider = new DefaultPdProvider(pdAddress); + hgCmdClient.init(new RpcOptions(), new HgCmdClient.PartitionAgent() { + @Override + public Endpoint getPartitionLeader(String graphName, int partitionId) { + Metapb.Shard shard = pdProvider.getPartitionLeader(graphName, partitionId); + return JRaftUtils.getEndPoint( + pdProvider.getStoreByID(shard.getStoreId()).getRaftAddress()); + } + }); + + storeClient = HgStoreClient.create(PDConfig.of(pdAddress) + .setEnableCache(true)); + HgStoreSession session = storeClient.openSession(graphName); + pdClient = storeClient.getPdClient(); + session.createTable(tableName); + String createGraph = "create_graph"; + HgOwnerKey hgOwnerKey = toOwnerKey(createGraph); + // 需要写数据，才会创建图 + session.put(tableName, + hgOwnerKey, createGraph.getBytes(StandardCharsets.UTF_8)); + Assert.assertEquals(createGraph, toStr(session.get(tableName, hgOwnerKey))); + + Integer partId = 0; + + Metapb.Partition pt = Metapb.Partition.newBuilder().build(); + CleanDataRequest request = new CleanDataRequest(); + request.setGraphName(graphName); + request.setPartitionId(partId); + + try { + CleanDataResponse response = hgCmdClient.cleanData(request); + if (response == null) { + log.error("response is null "); + } else if (response.getStatus() == null) { + log.error("response status is null"); + } + + log.info("response status:{} ", response.getStatus()); + + Assert.assertTrue(response.getStatus().isOK()); + } catch (Exception e) { + e.printStackTrace(); + } + } + + // @Test + // TODO: partition not local + public void testUpdatePartitionLeader() { + hgCmdClient = new HgCmdClient(); + pdProvider = new DefaultPdProvider(pdAddress); + hgCmdClient.init(new RpcOptions(), new HgCmdClient.PartitionAgent() { + @Override + public Endpoint getPartitionLeader(String graphName, int partitionId) { + Metapb.Shard shard = pdProvider.getPartitionLeader(graphName, partitionId); + return JRaftUtils.getEndPoint( + pdProvider.getStoreByID(shard.getStoreId()).getRaftAddress()); + } + }); + + storeClient = HgStoreClient.create(PDConfig.of(pdAddress) + .setEnableCache(true)); + HgStoreSession session = storeClient.openSession(graphName); + pdClient = storeClient.getPdClient(); + session.createTable(tableName); + String createGraph = "create_graph"; + HgOwnerKey hgOwnerKey = toOwnerKey(createGraph); + // 需要写数据，才会创建图 + session.put(tableName, + hgOwnerKey, createGraph.getBytes(StandardCharsets.UTF_8)); + Assert.assertEquals(createGraph, toStr(session.get(tableName, hgOwnerKey))); + + } + + // @Test + // TODO: partition not local + public void testData() { + hgCmdClient = new HgCmdClient(); + pdProvider = new DefaultPdProvider(pdAddress); + hgCmdClient.init(new RpcOptions(), new HgCmdClient.PartitionAgent() { + @Override + public Endpoint getPartitionLeader(String graphName, int partitionId) { + Metapb.Shard shard = pdProvider.getPartitionLeader(graphName, partitionId); + return JRaftUtils.getEndPoint( + pdProvider.getStoreByID(shard.getStoreId()).getRaftAddress()); + } + }); + + storeClient = HgStoreClient.create(PDConfig.of(pdAddress) + .setEnableCache(true)); + HgStoreSession session = storeClient.openSession("hugegraphtest"); + pdClient = storeClient.getPdClient(); + session.truncate(); + + int loop = 3; + + for (int i = 0; i < loop; i++) { + HgOwnerKey hgOwnerKey = toOwnerKey(i + "owner:" + i, i + "k:" + i); + session.put(tableName, hgOwnerKey, toIntBytes(i)); + } + + try { + HgKvIterator iterable = session.scanIterator(tableName); + int x = 0; + while (iterable.hasNext()) { + HgKvEntry entry = iterable.next(); + log.info("data:{}-{}", toStr(entry.key()), entry.value()); + x++; + } + log.info("x={}", x); + Assert.assertEquals(x, loop); + } catch (Exception e) { + e.printStackTrace(); + } + } + + // @Test + // TODO: partition not local + public void testCompressionData() { + + hgCmdClient = new HgCmdClient(); + pdProvider = new DefaultPdProvider(pdAddress); + hgCmdClient.init(new RpcOptions(), new HgCmdClient.PartitionAgent() { + @Override + public Endpoint getPartitionLeader(String graphName, int partitionId) { + Metapb.Shard shard = pdProvider.getPartitionLeader(graphName, partitionId); + return JRaftUtils.getEndPoint( + pdProvider.getStoreByID(shard.getStoreId()).getRaftAddress()); + } + }); + + storeClient = HgStoreClient.create(PDConfig.of(pdAddress) + .setEnableCache(true)); + HgStoreSession session = storeClient.openSession("hugegraphtest"); + pdClient = storeClient.getPdClient(); + session.truncate(); + + int loop = 10; + + for (int i = 0; i < loop; i++) { + String key = "d41d8cd98f00b204e9800998ecf8427e" + getMd5("a" + i) + getId(); + String value = "10000" + getId() + getId(); + HgOwnerKey hgOwnerKey = toOwnerKey("d41d8cd98f00b204e9800998ecf8427e", key); + session.put(tableName, hgOwnerKey, value.getBytes()); + } + + try { + HgKvIterator iterable = session.scanIterator(tableName); + int x = 0; + while (iterable.hasNext()) { + HgKvEntry entry = iterable.next(); + x++; + } + log.info("x={}", x); + Assert.assertEquals(x, loop); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/MetricServiceTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/MetricServiceTest.java new file mode 100644 index 0000000000..bc7ff64021 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/MetricServiceTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core; + +import java.util.Map; + +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.metric.HgMetricService; +import org.apache.hugegraph.store.metric.SystemMetricService; +import org.junit.Before; +import org.junit.Test; + +public class MetricServiceTest { + + private SystemMetricService service; + private HgMetricService hgService; + + @Before + public void setUp() { + service = new SystemMetricService(); + HgStoreEngine instance = HgStoreEngine.getInstance(); + service.setStoreEngine(instance); + hgService = HgMetricService.getInstance().setHgStoreEngine(instance); + } + + @Test + public void testGetStorageEngine() { + HgStoreEngine result = service.getStorageEngine(); + } + + @Test + public void testGetSystemMetrics() { + try { + Map systemMetrics = service.getSystemMetrics(); + Thread.sleep(1000); + systemMetrics = service.getSystemMetrics(); + } catch (Exception e) { + + } + } + + @Test + public void testGetHgMetrics() { + // Setup + Metapb.StoreStats.Builder systemMetrics = hgService.getMetrics(); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/StoreEngineTestBase.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/StoreEngineTestBase.java new file mode 100644 index 0000000000..267b5a566d --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/StoreEngineTestBase.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core; + +import java.io.File; +import java.util.HashMap; + +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.PartitionEngine; +import org.apache.hugegraph.store.UnitTestBase; +import org.apache.hugegraph.store.business.DefaultDataMover; +import org.apache.hugegraph.store.meta.Partition; +import org.apache.hugegraph.store.meta.ShardGroup; +import org.apache.hugegraph.store.options.HgStoreEngineOptions; +import org.apache.hugegraph.store.options.RaftRocksdbOptions; +import org.apache.hugegraph.store.pd.FakePdServiceProvider; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import com.alipay.sofa.jraft.util.StorageOptionsFactory; + +import lombok.extern.slf4j.Slf4j; + +/** + * 使用 FakePd 和 FakePdOptions，初始化 HgStoreEngine，该类的各项 get 函数可用 + */ +@Slf4j +public class StoreEngineTestBase { + + private static final String DB_PATH = "/tmp/junit"; + private static int initCount = 0; + + @BeforeClass + public static void initEngine() { + UnitTestBase.deleteDir(new File(DB_PATH)); + + HgStoreEngineOptions options = new HgStoreEngineOptions(); + options.setDataPath(DB_PATH); + options.setRaftPath(DB_PATH); + options.setFakePD(true); + options.setRocksdbConfig(new HashMap<>() {{ + put("rocksdb.write_buffer_size", "1048576"); + }}); + options.setGrpcAddress("127.0.0.1:6511"); + options.setRaftAddress("127.0.0.1:6510"); + options.setDataTransfer(new DefaultDataMover()); + + options.setFakePdOptions(new HgStoreEngineOptions.FakePdOptions() {{ + setStoreList("127.0.0.1"); + setPeersList("127.0.0.1"); + setPartitionCount(1); + setShardCount(1); + }}); + + if (initCount == 0) { + StorageOptionsFactory.releaseAllOptions(); + RaftRocksdbOptions.initRocksdbGlobalConfig(options.getRocksdbConfig()); + initCount++; + } + HgStoreEngine.getInstance().init(options); + } + + public static Partition getPartition(int partitionId) { + return getPartition(partitionId, "graph0"); + } + + public static Partition getPartition(int partitionId, String graphName) { + Partition partition = new Partition(); + partition.setId(partitionId); + partition.setGraphName(graphName); + partition.setStartKey(0); + partition.setEndKey(65535); + partition.setWorkState(Metapb.PartitionState.PState_Normal); + partition.setVersion(1); + return partition; + } + + /** + * 创建 分区为 0 的 partition engine. 该分区 1 个 shard，为 leader, graph name: graph0 + * + * @return + */ + public static PartitionEngine createPartitionEngine(int partitionId) { + return createPartitionEngine(partitionId, "graph0"); + } + + public static PartitionEngine createPartitionEngine(int partitionId, String graphName) { + Metapb.Shard shard = Metapb.Shard.newBuilder() + .setStoreId(FakePdServiceProvider.makeStoreId( + "127.0.0.1:6511")) + .setRole(Metapb.ShardRole.Leader) + .build(); + + Metapb.ShardGroup shardGroup = Metapb.ShardGroup.newBuilder() + .setId(partitionId) + .setConfVer(1) + .setVersion(1) + .setState( + Metapb.PartitionState.PState_Normal) + .addShards(shard) + .build(); + + getStoreEngine().getPartitionManager().updateShardGroup(ShardGroup.from(shardGroup)); + + var engine = getStoreEngine().createPartitionEngine(getPartition(partitionId, graphName)); + engine.waitForLeader(2000); + return engine; + } + + public static HgStoreEngine getStoreEngine() { + return HgStoreEngine.getInstance(); + } + + @AfterClass + public static void shutDownEngine() { + try { + HgStoreEngine.getInstance().shutdown(); + } catch (Exception e) { + log.error("shut down engine error: {}", e.getMessage()); + } + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/HgStoreStateMachineTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/HgStoreStateMachineTest.java new file mode 100644 index 0000000000..752a17ea59 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/HgStoreStateMachineTest.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core.raft; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hugegraph.store.raft.HgStoreStateMachine; +import org.apache.hugegraph.store.raft.RaftClosure; +import org.apache.hugegraph.store.raft.RaftOperation; +import org.apache.hugegraph.store.raft.RaftStateListener; +import org.apache.hugegraph.store.raft.RaftTaskHandler; +import org.apache.hugegraph.store.snapshot.HgSnapshotHandler; +import org.apache.hugegraph.store.util.HgStoreException; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.Iterator; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.conf.Configuration; +import com.alipay.sofa.jraft.entity.LeaderChangeContext; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.entity.Task; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.error.RaftException; + +@RunWith(MockitoJUnitRunner.class) +public class HgStoreStateMachineTest { + + @Mock + private HgSnapshotHandler mockSnapshotHandler; + + private HgStoreStateMachine hgStoreStateMachineUnderTest; + + @Before + public void setUp() { + hgStoreStateMachineUnderTest = new HgStoreStateMachine(0, mockSnapshotHandler); + } + + @Test + public void testAddTaskHandler() { + // Setup + final RaftTaskHandler handler = new RaftTaskHandler() { + @Override + public boolean invoke(int groupId, byte[] request, RaftClosure response) throws + HgStoreException { + return false; + } + + @Override + public boolean invoke(int groupId, byte methodId, Object req, RaftClosure response) + throws HgStoreException { + return false; + } + }; + + // Run the test + hgStoreStateMachineUnderTest.addTaskHandler(handler); + + // Verify the results + } + + @Test + public void testAddStateListener() { + // Setup + final RaftStateListener mockListener = new RaftStateListener() { + @Override + public void onLeaderStart(long newTerm) { + + } + + @Override + public void onError(RaftException e) { + + } + }; + + // Run the test + hgStoreStateMachineUnderTest.addStateListener(mockListener); + + // Verify the results + } + + @Test + public void testIsLeader() { + // Setup + // Run the test + final boolean result = hgStoreStateMachineUnderTest.isLeader(); + + // Verify the results + assertFalse(result); + } + + @Test + public void testOnApply() { + RaftOperation op = RaftOperation.create((byte) 0b0); + final Task task = new Task(); + task.setData(ByteBuffer.wrap(op.getValues())); + task.setDone(new HgStoreStateMachine.RaftClosureAdapter(op, closure -> { + + })); + + List tasks = new ArrayList<>(); + tasks.add(task); + // Setup + final Iterator inter = new Iterator() { + final java.util.Iterator iterator = tasks.iterator(); + Task task; + + @Override + public ByteBuffer getData() { + return task.getData(); + } + + @Override + public long getIndex() { + return 0; + } + + @Override + public long getTerm() { + return 0; + } + + @Override + public Closure done() { + return null; + } + + @Override + public void setErrorAndRollback(long ntail, Status st) { + + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public ByteBuffer next() { + task = iterator.next(); + return task.getData(); + } + }; + + // Run the test + hgStoreStateMachineUnderTest.onApply(inter); + + // Verify the results + } + + @Test + public void testGetLeaderTerm() { + // Setup + // Run the test + final long result = hgStoreStateMachineUnderTest.getLeaderTerm(); + + // Verify the results + assertEquals(-1L, result); + } + + + @Test + public void testOnLeaderStart() { + // Setup + // Run the test + hgStoreStateMachineUnderTest.onLeaderStart(0L); + + // Verify the results + } + + @Test + public void testOnLeaderStop() { + // Setup + final Status status = new Status(RaftError.UNKNOWN, "fmt", "args"); + + // Run the test + hgStoreStateMachineUnderTest.onLeaderStop(status); + + // Verify the results + } + + @Test + public void testOnStartFollowing() { + // TODO: uncomment later (jraft) +// // Setup +// final LeaderChangeContext ctx = +// new LeaderChangeContext(new PeerId("ip", 0, 0, 0), "groupId", 0L, +// new Status(RaftError.UNKNOWN, "fmt", "args")); +// +// // Run the test +// hgStoreStateMachineUnderTest.onStartFollowing(ctx); + + // Verify the results + } + + @Test + public void testOnStopFollowing() { + // TODO: uncomment later (jraft) +// // Setup +// final LeaderChangeContext ctx = +// new LeaderChangeContext(new PeerId("ip", 0, 0, 0), "groupId", 0L, +// new Status(RaftError.UNKNOWN, "fmt", "args")); +// +// // Run the test +// hgStoreStateMachineUnderTest.onStopFollowing(ctx); + + // Verify the results + } + + @Test + public void testOnConfigurationCommitted() { + // Setup + final Configuration conf = new Configuration(List.of(new PeerId("ip", 0, 0, 0)), + List.of(new PeerId("ip", 0, 0, 0))); + + // Run the test + hgStoreStateMachineUnderTest.onConfigurationCommitted(conf); + + // Verify the results + } + +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/MockStateMachine.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/MockStateMachine.java new file mode 100644 index 0000000000..2af5c683c9 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/MockStateMachine.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core.raft; + +import com.alipay.sofa.jraft.Iterator; +import com.alipay.sofa.jraft.core.StateMachineAdapter; + +public class MockStateMachine extends StateMachineAdapter { + @Override + public void onApply(Iterator iter) { + while (iter.hasNext()) { + iter.next(); + } + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/RaftOperationTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/RaftOperationTest.java new file mode 100644 index 0000000000..586c2d8c10 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/RaftOperationTest.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core.raft; + +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.store.raft.RaftOperation; +import org.junit.Before; +import org.junit.Test; + +import com.google.protobuf.GeneratedMessageV3; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +public class RaftOperationTest { + + private RaftOperation raftOperationUnderTest; + + @Before + public void setUp() { + raftOperationUnderTest = new RaftOperation(); + } + + + @Test + public void testCreate1() { + // Run the test + final RaftOperation result = RaftOperation.create((byte) 0b0); + assertEquals(null, result.getReq()); + assertEquals((byte) 0b0, result.getOp()); + } + + @Test + public void testCreate2() { + // Run the test + final RaftOperation result = RaftOperation.create((byte) 0b0, "content".getBytes(), "req"); + assertArrayEquals("content".getBytes(), result.getValues()); + assertEquals("req", result.getReq()); + assertEquals((byte) 0b0, result.getOp()); + } + + @Test + public void testCreate3() { + // Run the test + final RaftOperation result = RaftOperation.create((byte) 0b0, "req"); + assertEquals("req", result.getReq()); + assertEquals((byte) 0b0, result.getOp()); + } + + @Test + public void testCreate4() throws Exception { + // Setup + final GeneratedMessageV3 req = Metapb.Graph.newBuilder().setGraphName("name").build(); + + // Run the test + final RaftOperation result = RaftOperation.create((byte) 0b0, req); + assertEquals((byte) 0b0, result.getOp()); + + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/RaftUtilsTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/RaftUtilsTest.java new file mode 100644 index 0000000000..9f8b72b3b1 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/RaftUtilsTest.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core.raft; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.util.Collections; +import java.util.List; + +import org.apache.hugegraph.store.UnitTestBase; +import org.apache.hugegraph.store.raft.util.RaftUtils; +import org.junit.Before; +import org.junit.Test; + +import com.alipay.sofa.jraft.Node; +import com.alipay.sofa.jraft.NodeManager; +import com.alipay.sofa.jraft.conf.Configuration; +import com.alipay.sofa.jraft.core.NodeImpl; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.option.NodeOptions; +import com.alipay.sofa.jraft.util.Endpoint; + +public class RaftUtilsTest { + final String ip = "127.0.0.1"; + final int port = 12456; + final String dataPath = "tmp/raftUtils"; + + @Before + public void setUp() { + + UnitTestBase.deleteDir(new File(dataPath)); + new File(dataPath).mkdirs(); + } + + @Test + public void testGetAllEndpoints1() throws InterruptedException { + final Endpoint addr = new Endpoint(ip, port); + final PeerId peer = new PeerId(addr, 0); + NodeManager.getInstance().addAddress(addr); + final NodeOptions nodeOptions = new NodeOptions(); + nodeOptions.setFsm(new MockStateMachine()); + nodeOptions.setLogUri(this.dataPath + File.separator + "log"); + nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta"); + nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot"); + nodeOptions.setInitialConf(new Configuration(Collections.singletonList(peer))); + final Node node = new NodeImpl("unittest", new PeerId(addr, 0)); + assertTrue(node.init(nodeOptions)); + node.isLeader(true); + + // Run the test + final List result = RaftUtils.getAllEndpoints(node); + // Verify the results + assertEquals(1, result.size()); + node.shutdown(); + node.join(); + + } + + @Test + public void testGetAllEndpoints2() { + final Endpoint addr = new Endpoint(ip, port); + final PeerId peer = new PeerId(addr, 0); + + Configuration conf = new Configuration(Collections.singletonList(peer)); + // Run the test + final List result = RaftUtils.getAllEndpoints(conf); + + // Verify the results + assertEquals(1, result.size()); + } + + @Test + public void testGetPeerEndpoints1() throws InterruptedException { + final Endpoint addr = new Endpoint(ip, port); + final PeerId peer = new PeerId(addr, 0); + NodeManager.getInstance().addAddress(addr); + final NodeOptions nodeOptions = new NodeOptions(); + nodeOptions.setFsm(new MockStateMachine()); + nodeOptions.setLogUri(this.dataPath + File.separator + "log"); + nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta"); + nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot"); + nodeOptions.setInitialConf(new Configuration(Collections.singletonList(peer))); + final Node node = new NodeImpl("unittest", new PeerId(addr, 0)); + assertTrue(node.init(nodeOptions)); + node.isLeader(true); + + + // Run the test + final List result = RaftUtils.getPeerEndpoints(node); + // Verify the results + assertEquals(1, result.size()); + node.shutdown(); + node.join(); + } + + @Test + public void testGetPeerEndpoints2() { + // Setup + final Configuration conf = new Configuration(List.of(new PeerId("ip", 0, 0, 0)), + List.of(new PeerId("ip", 0, 0, 0))); + + // Run the test + final List result = RaftUtils.getPeerEndpoints(conf); + + // Verify the results + assertEquals(1, result.size()); + } + + @Test + public void testGetLearnerEndpoints1() throws InterruptedException { + // Setup + final Endpoint addr = new Endpoint(ip, port); + final PeerId peer = new PeerId(addr, 0); + final PeerId peer2 = new PeerId(new Endpoint(ip, 13456), 0); + NodeManager.getInstance().addAddress(addr); + final NodeOptions nodeOptions = new NodeOptions(); + nodeOptions.setFsm(new MockStateMachine()); + nodeOptions.setLogUri(this.dataPath + File.separator + "log"); + nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta"); + nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot"); + nodeOptions.setInitialConf(new Configuration(Collections.singletonList(peer), + Collections.singletonList(peer2))); + final Node node = new NodeImpl("unittest", new PeerId(addr, 0)); + assertTrue(node.init(nodeOptions)); + node.isLeader(true); + // Run the test + final List result = RaftUtils.getLearnerEndpoints(node); + + // Verify the results + assertEquals(1, result.size()); + node.shutdown(); + node.join(); + } + + @Test + public void testGetLearnerEndpoints2() { + // Setup + final Configuration conf = new Configuration(List.of(new PeerId("ip", 0, 0, 0)), + List.of(new PeerId("ip", 0, 0, 0))); + + // Run the test + final List result = RaftUtils.getLearnerEndpoints(conf); + + // Verify the results + assertEquals(1, result.size()); + } + + @Test + public void testConfigurationEquals() { + // Setup + final Configuration oldConf = new Configuration(List.of(new PeerId("ip", 0, 0, 0)), + List.of(new PeerId("ip", 0, 0, 0))); + final Configuration newConf = new Configuration(List.of(new PeerId("ip", 0, 0, 0)), + List.of(new PeerId("ip", 0, 0, 0))); + + // Run the test + final boolean result = RaftUtils.configurationEquals(oldConf, newConf); + + // Verify the results + assertTrue(result); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/snapshot/HgSnapshotHandlerTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/snapshot/HgSnapshotHandlerTest.java new file mode 100644 index 0000000000..ff5ef24acf --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/snapshot/HgSnapshotHandlerTest.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core.snapshot; + +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.io.FileUtils; +import org.apache.hugegraph.store.core.StoreEngineTestBase; +import org.apache.hugegraph.store.meta.Partition; +import org.apache.hugegraph.store.snapshot.HgSnapshotHandler; +import org.junit.Before; +import org.junit.Test; + +import com.alipay.sofa.jraft.entity.RaftOutter; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; +import com.google.protobuf.Message; + + +public class HgSnapshotHandlerTest extends StoreEngineTestBase { + + private static HgSnapshotHandler hgSnapshotHandlerUnderTest; + + @Before + public void setUp() throws IOException { + hgSnapshotHandlerUnderTest = new HgSnapshotHandler(createPartitionEngine(0)); + FileUtils.forceMkdir(new File("/tmp/snapshot")); + FileUtils.forceMkdir(new File("/tmp/snapshot/data")); + } + + @Test + public void testGetPartitions() { + // Run the test + final Map result = hgSnapshotHandlerUnderTest.getPartitions(); + // Verify the results + assertEquals(1, result.size()); + } + + @Test + public void testOnSnapshotSaveAndLoad() { + String path = "/tmp/snapshot"; + // Setup + final SnapshotWriter writer = new SnapshotWriter() { + @Override + public boolean saveMeta(RaftOutter.SnapshotMeta meta) { + return false; + } + + @Override + public boolean addFile(String fileName, Message fileMeta) { + return false; + } + + @Override + public boolean removeFile(String fileName) { + return false; + } + + @Override + public void close(boolean keepDataOnError) throws IOException { + + } + + @Override + public boolean init(Void opts) { + return false; + } + + @Override + public void shutdown() { + + } + + @Override + public String getPath() { + return path; + } + + @Override + public Set listFiles() { + return null; + } + + @Override + public Message getFileMeta(String fileName) { + return null; + } + + @Override + public void close() throws IOException { + + } + }; + + // Run the test + hgSnapshotHandlerUnderTest.onSnapshotSave(writer); + + // Verify the results + + // Setup + final SnapshotReader reader = new SnapshotReader() { + final String path = "/tmp/snapshot"; + + @Override + public RaftOutter.SnapshotMeta load() { + return null; + } + + @Override + public String generateURIForCopy() { + return null; + } + + @Override + public boolean init(Void opts) { + return false; + } + + @Override + public void shutdown() { + + } + + @Override + public String getPath() { + return path; + } + + @Override + public Set listFiles() { + return null; + } + + @Override + public Message getFileMeta(String fileName) { + return null; + } + + @Override + public void close() throws IOException { + + } + }; + + // Run the test + hgSnapshotHandlerUnderTest.onSnapshotLoad(reader, 0L); + } + + + @Test + public void testTrimStartPath() { + assertEquals("str", HgSnapshotHandler.trimStartPath("str", "prefix")); + } + + @Test + public void testFindFileList() { + // Setup + final File dir = new File("filename.txt"); + final File rootDir = new File("filename.txt"); + + // Run the test + HgSnapshotHandler.findFileList(dir, rootDir, List.of("value")); + + // Verify the results + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/HgStoreEngineTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/HgStoreEngineTest.java new file mode 100644 index 0000000000..0bc54d4700 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/HgStoreEngineTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core.store; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.util.List; + +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.core.StoreEngineTestBase; +import org.apache.hugegraph.store.pd.FakePdServiceProvider; +import org.apache.hugegraph.testutil.Assert; +import org.junit.Before; +import org.junit.Test; + +public class HgStoreEngineTest extends StoreEngineTestBase { + + private HgStoreEngine engine; + + @Before + public void setup() { + engine = HgStoreEngine.getInstance(); + } + + @Test + public void testGetInstance() { + assertNotNull(HgStoreEngine.getInstance()); + } + + // @Test + // TODO: npe and not leader + public void testStateChanged() { + createPartitionEngine(0); + var storeId = FakePdServiceProvider.makeStoreId("127.0.0.1:6511"); + var store = engine.getPartitionManager().getStore(storeId); + engine.stateChanged(store, Metapb.StoreState.Offline, Metapb.StoreState.Up); + assertEquals(engine.getPartitionEngines().size(), 1); + } + + @Test + public void testCreatePartitionEngine() { + var partition = getPartition(0); + assertNotNull(engine.createPartitionEngine(partition)); + } + + @Test + public void testCreatePartitionGroups() { + var partition = getPartition(0); + engine.createPartitionGroups(partition); + } + + @Test + public void testDestroyPartitionEngine() { + createPartitionEngine(16); + engine.destroyPartitionEngine(16, List.of("graph0")); + // assertEquals(engine.getPartitionEngines().size(), 0); + } + + @Test + public void testDeletePartition() { + createPartitionEngine(0); + engine.deletePartition(0, "graph0"); + // TODO: check logic + assertEquals(engine.getPartitionEngines().size(), 1); + } + + @Test + public void testGetLeaderPartition() throws InterruptedException { + createPartitionEngine(0); + assertEquals(engine.getLeaderPartition().size(), 1); + } + + @Test + public void testGetAlivePeers() throws InterruptedException { + createPartitionEngine(0); + assertEquals(engine.getAlivePeers(0).size(), 1); + } + + @Test + public void testGetLeaderTerm() { + createPartitionEngine(0); + // no vote + assertEquals(engine.getLeaderTerm(0), -1); + } + + @Test + public void testGetCommittedIndex() throws InterruptedException { + createPartitionEngine(0); + // write something background + Assert.assertTrue(engine.getCommittedIndex(0) > 0); + } + + @Test + public void testGetRaftRpcServer() { + assertNotNull(engine.getRaftRpcServer()); + } + + @Test + public void testGetPartitionManager() { + assertNotNull(engine.getPartitionManager()); + } + + @Test + public void testGetDataMover() { + assertNotNull(engine.getDataMover()); + } + + @Test + public void testGetPdProvider() { + assertNotNull(engine.getPdProvider()); + } + + @Test + public void testGetCmdClient() { + assertNotNull(engine.getHgCmdClient()); + } + + @Test + public void testGetHeartbeatService() { + assertNotNull(engine.getHeartbeatService()); + } + + @Test + public void testIsClusterReady() throws InterruptedException { + // wait heart beat + Thread.sleep(2000); + assertNotNull(engine.isClusterReady()); + } + + @Test + public void testGetDataLocations() { + assertEquals(engine.getDataLocations().size(), 1); + } + + @Test + public void testGetPartitionEngine() { + createPartitionEngine(0); + assertNotNull(engine.getPartitionEngine(0)); + } + + @Test + public void testGetPartitionEngines() { + createPartitionEngine(0); + assertEquals(engine.getPartitionEngines().size(), 1); + } + + @Test + public void testGetNodeMetrics() { + assertNotNull(engine.getNodeMetrics()); + } + + @Test + public void testGetRaftGroupCount() { + createPartitionEngine(0); + assertEquals(engine.getRaftGroupCount(), 1); + } + +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/PartitionEngineTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/PartitionEngineTest.java new file mode 100644 index 0000000000..02ff99259a --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/PartitionEngineTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core.store; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import org.apache.hugegraph.store.PartitionEngine; +import org.apache.hugegraph.store.core.StoreEngineTestBase; +import org.junit.Before; +import org.junit.Test; + +public class PartitionEngineTest extends StoreEngineTestBase { + + PartitionEngine engine; + + @Before + public void setup() { + engine = createPartitionEngine(0); + } + + @Test + public void testHasPartition() { + assertTrue(engine.hasPartition("graph0")); + } + + @Test + public void testGetGroupId() { + assertEquals(engine.getGroupId().intValue(), 0); + } + + @Test + public void testGetShardGroup() { + assertEquals(engine.getShardGroup().getShards().size(), 1); + } + + @Test + public void testIsLeader() { + System.out.println(engine.isLeader()); + } + + @Test + public void testGetLeader() { + assertEquals(engine.getLeader().toString(), engine.getOptions().getRaftAddress()); + } + + @Test + public void testGetAlivePeers() { + try { + System.out.println(engine.getAlivePeers().size()); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Test + public void testGetRaftNode() { + assertNotNull(engine.getRaftNode()); + } + + @Test + public void testGetPartitions() { + assertEquals(engine.getPartitions().get("graph0").getId(), 0); + } + + @Test + public void testGetPartition() { + assertEquals(engine.getPartition("graph0").getId(), 0); + } + + @Test + public void testGetCommittedIndex() throws InterruptedException { + Thread.sleep(1000); + System.out.println(engine.getCommittedIndex()); + } + +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/PartitionInstructionProcessorTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/PartitionInstructionProcessorTest.java new file mode 100644 index 0000000000..7a0aae85de --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/PartitionInstructionProcessorTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core.store; + +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.pd.grpc.pulse.DbCompaction; +import org.apache.hugegraph.pd.grpc.pulse.SplitPartition; +import org.apache.hugegraph.pd.grpc.pulse.TransferLeader; +import org.apache.hugegraph.store.PartitionInstructionProcessor; +import org.apache.hugegraph.store.core.StoreEngineTestBase; +import org.apache.hugegraph.store.pd.FakePdServiceProvider; +import org.junit.Before; +import org.junit.Test; + +public class PartitionInstructionProcessorTest extends StoreEngineTestBase { + + PartitionInstructionProcessor processor; + + @Before + public void init() { + processor = new PartitionInstructionProcessor(getStoreEngine()); + } + + @Test + public void testTransferLeader() { + var engine = createPartitionEngine(0); + engine.waitForLeader(1000); + var shard = Metapb.Shard.newBuilder() + .setStoreId(FakePdServiceProvider.makeStoreId("127.0.0.1:6511")) + .setRole(Metapb.ShardRole.Leader) + .build(); + + TransferLeader trans = TransferLeader.newBuilder() + .setShard(shard) + .build(); + processor.onTransferLeader(1, getPartition(0), trans, integer -> { + assertEquals(0, integer.intValue()); + }); + } + + @Test + public void testDbCompaction() throws InterruptedException { + var engine = createPartitionEngine(0); + engine.waitForLeader(1000); + DbCompaction dbCompaction = DbCompaction.newBuilder() + .setTableName("test") + .build(); + processor.onDbCompaction(2, getPartition(0), dbCompaction, integer -> { + assertEquals(0, integer.intValue()); + }); + + Thread.sleep(2000); + } + + @Test + public void testSplitPartition() throws InterruptedException { + var engine = createPartitionEngine(0); + engine.waitForLeader(1000); + + var partition = getPartition(0); + System.out.println(partition); + + List list = new ArrayList<>(); + list.add(Metapb.Partition.newBuilder(partition.getProtoObj()) + .setStartKey(0) + .setEndKey(20000) + .build()); + list.add(Metapb.Partition.newBuilder(partition.getProtoObj()) + .setStartKey(20000) + .setEndKey(65535) + .setId(1).build()); + + SplitPartition splitPartition = SplitPartition.newBuilder() + .addAllNewPartition(list) + .build(); + + processor.onSplitPartition(3, partition, splitPartition, integer -> { + assertEquals(0, integer.intValue()); + }); + + Thread.sleep(2000); + System.out.println(getStoreEngine().getPartitionEngines()); + } + + @Test + public void testChangeShard() { + + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/meta/MetadataKeyHelperTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/meta/MetadataKeyHelperTest.java new file mode 100644 index 0000000000..768d0370fa --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/meta/MetadataKeyHelperTest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core.store.meta; + +import static org.junit.Assert.assertArrayEquals; + +import org.apache.hugegraph.store.meta.MetadataKeyHelper; +import org.junit.Test; + +public class MetadataKeyHelperTest { + + @Test + public void testKey() { + assertArrayEquals("HUGEGRAPH/TASK/".getBytes(), MetadataKeyHelper.getTaskPrefix()); + assertArrayEquals("HUGEGRAPH/TASK/0/".getBytes(), MetadataKeyHelper.getTaskPrefix(0)); + assertArrayEquals("HUGEGRAPH/TASK_DONE/0000000000000000".getBytes(), + MetadataKeyHelper.getDoneTaskKey(0)); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/meta/PartitionManagerTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/meta/PartitionManagerTest.java new file mode 100644 index 0000000000..0cdee7333a --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/meta/PartitionManagerTest.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core.store.meta; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.store.cmd.UpdatePartitionRequest; +import org.apache.hugegraph.store.core.StoreEngineTestBase; +import org.apache.hugegraph.store.meta.Graph; +import org.apache.hugegraph.store.meta.GraphManager; +import org.apache.hugegraph.store.meta.PartitionManager; +import org.apache.hugegraph.store.pd.FakePdServiceProvider; +import org.junit.Before; +import org.junit.Test; + +public class PartitionManagerTest extends StoreEngineTestBase { + + private PartitionManager manager; + + @Before + public void setup() { + manager = getStoreEngine().getPartitionManager(); + } + + @Test + public void testGetDeletedFileManager() { + assertNotNull(manager.getDeletedFileManager()); + } + + @Test + public void testGetPdProvider() { + assertEquals(manager.getPdProvider().getClass(), FakePdServiceProvider.class); + } + + @Test + public void testGetStoreMetadata() { + assertNotNull(manager.getStoreMetadata()); + } + + @Test + public void testSetStore() { + var storeId = FakePdServiceProvider.makeStoreId("127.0.0.1:6511"); + var store = manager.getStore(storeId); + manager.setStore(store); + var store2 = manager.getStoreMetadata().getStore(); + + assertEquals(store.getId(), store2.getId()); + } + + @Test + public void testUpdatePartition() { + var partition = getPartition(5); + manager.updatePartition(partition.getProtoObj(), true); + + var partition2 = manager.findPartition("graph0", 5); + assertEquals(partition.getGraphName(), partition2.getGraphName()); + + var partition3 = manager.loadPartitionFromSnapshot(partition2); + assertEquals(partition3.getGraphName(), partition2.getGraphName()); + } + + @Test + public void testChangeState() { + createPartitionEngine(4); + var partition = getPartition(4); + manager.changeState(partition, Metapb.PartitionState.PState_Offline); + var partition2 = manager.findPartition("graph0", 4); + assertEquals(partition2.getWorkState(), Metapb.PartitionState.PState_Offline); + } + + @Test + public void testChangeKeyRange() { + createPartitionEngine(4); + var partition = getPartition(4); + manager.changeKeyRange(partition, 1000, 2000); + + var partition2 = manager.findPartition("graph0", 4); + assertEquals(partition2.getStartKey(), 1000); + assertEquals(partition2.getEndKey(), 2000); + } + + + @Test + public void testUpdatePartitionRangeOrState() { + createPartitionEngine(4); + UpdatePartitionRequest request = new UpdatePartitionRequest(); + request.setPartitionId(4); + request.setGraphName("graph0"); + request.setStartKey(2000); + request.setEndKey(3000); + request.setWorkState(Metapb.PartitionState.PState_Offline); + manager.updatePartitionRangeOrState(request); + + var partition = manager.findPartition("graph0", 4); + assertEquals(partition.getStartKey(), 2000); + assertEquals(partition.getEndKey(), 3000); + assertEquals(partition.getWorkState(), Metapb.PartitionState.PState_Offline); + } + + @Test + public void testGetLeaderPartitionIds() { + createPartitionEngine(0); + createPartitionEngine(4); + createPartitionEngine(5); + System.out.println(manager.getLeaderPartitionIds("graph0")); + assertEquals(manager.getLeaderPartitionIds("graph0").size(), 3); + } + + @Test + public void testisLocal() { + createPartitionEngine(0); + assertTrue(manager.isLocalPartition(0)); + assertTrue(manager.isLocalPartition(getPartition(0))); + assertTrue(manager.isLocalStore(manager.getStore())); + } + + @Test + public void testUploadToPd() throws PDException { + createPartitionEngine(0); + var partition = manager.findPartition("graph0", 0); + var list = new ArrayList(); + list.add(partition.getProtoObj()); + // fake pd, return nothing + assertEquals(1, manager.updatePartitionToPD(list).size()); + + manager.reportTask(null); + + var partitions = manager.changePartitionToOnLine(list); + assertSame(partitions.get(0).getState(), Metapb.PartitionState.PState_Normal); + // fake pd + // TODO: uncomment it until fix it + // assertNotNull(manager.findPartition("graph0", 1000)); + + } + + @Test + public void testShards2Peers() { + var storeId = FakePdServiceProvider.makeStoreId("127.0.0.1:6511"); + Metapb.Shard shard = Metapb.Shard.newBuilder() + .setStoreId(storeId) + .setRole(Metapb.ShardRole.Leader) + .build(); + + List list = new ArrayList<>(); + list.add(shard); + + var peers = manager.shards2Peers(list); + assertEquals("127.0.0.1:6510", peers.get(0)); + } + + @Test + public void testLoad() { + createPartitionEngine(0); + var graphManager = new GraphManager(manager.getOptions(), manager.getPdProvider()); + var graph = Metapb.Graph.newBuilder() + .setGraphName("graph0") + .setPartitionCount(12) + .build(); + graphManager.updateGraph(new Graph(graph)); + + manager.load(); + assertNotNull(manager.getLeaderPartitionIds("graph0")); + } + + @Test + public void testSyncPartitionsFromPD() throws PDException { + createPartitionEngine(0); + // from fake pd + manager.syncPartitionsFromPD(partition -> { + }); + + assertTrue(manager.getPartitions().isEmpty()); + } + +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/meta/TaskManagerTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/meta/TaskManagerTest.java new file mode 100644 index 0000000000..ac4b44f537 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/meta/TaskManagerTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core.store.meta; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.apache.hugegraph.pd.grpc.MetaTask; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.store.core.StoreEngineTestBase; +import org.apache.hugegraph.store.meta.TaskManager; +import org.apache.hugegraph.store.meta.asynctask.AsyncTaskState; +import org.apache.hugegraph.store.meta.asynctask.CleanTask; +import org.junit.Before; +import org.junit.Test; + +public class TaskManagerTest extends StoreEngineTestBase { + + private TaskManager manager; + + @Before + public void setup() { + createPartitionEngine(0, "graph0"); + manager = getStoreEngine().getPartitionEngine(0).getTaskManager(); + } + + @Test + public void testAsyncTaskReadAndWrite() { + var task1 = new CleanTask(0, "graph0", AsyncTaskState.START, null); + var task2 = new CleanTask(0, "graph0", AsyncTaskState.FAILED, null); + var task3 = new CleanTask(0, "graph0", AsyncTaskState.SUCCESS, null); + var task4 = new CleanTask(0, "graph0", AsyncTaskState.FAILED, null); + + manager.putAsyncTask(task1); + manager.putAsyncTask(task2); + manager.putAsyncTask(task3); + manager.putAsyncTask(task4); + + var list = manager.scanAsyncTasks(0, "graph0"); + assertEquals(list.size(), 4); + + var newTask1 = (CleanTask) manager.getOneAsyncTask(0, "graph0", task1.getId()); + assertEquals(task1.getState(), newTask1.getState()); + assertEquals(task1.getType(), newTask1.getType()); + + manager.updateAsyncTaskState(0, "graph0", task4.getId(), AsyncTaskState.SUCCESS); + var newTask4 = (CleanTask) manager.getOneAsyncTask(0, "graph0", task4.getId()); + assertEquals(newTask4.getState(), AsyncTaskState.SUCCESS); + + assertNull(manager.getOneAsyncTask(1, "graph0", "")); + } + + @Test + public void testTaskOp() { + MetaTask.Task task1 = MetaTask.Task.newBuilder() + .setId(1) + .setState(MetaTask.TaskState.Task_Ready) + .setType(MetaTask.TaskType.Split_Partition) + .setPartition(Metapb.Partition.newBuilder() + .setGraphName("graph0") + .setId(0).build()) + .build(); + + manager.updateTask(task1); + assertTrue(manager.taskExists(task1)); + assertFalse(manager.taskExists(0, "graph0", + MetaTask.TaskType.Split_Partition.name())); + assertFalse(manager.partitionTaskRepeat(0, "graph0", + MetaTask.TaskType.Split_Partition.name())); + + MetaTask.Task task2 = MetaTask.Task.newBuilder(task1).setId(2).build(); + manager.updateTask(task2); + + assertTrue(manager.taskExists(0, "graph0", + MetaTask.TaskType.Split_Partition.name())); + assertTrue(manager.partitionTaskRepeat(0, "graph0", + MetaTask.TaskType.Split_Partition.name())); + + MetaTask.Task task3 = MetaTask.Task.newBuilder(task1) + .setId(3) + .setState(MetaTask.TaskState.Task_Success) + .setPartition( + Metapb.Partition.newBuilder(task1.getPartition()) + .setGraphName("graph1") + .setId(1) + .build()) + .build(); + manager.updateTask(task3); + assertFalse(manager.taskExists(task3)); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/meta/asynctask/CleanTaskTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/meta/asynctask/CleanTaskTest.java new file mode 100644 index 0000000000..816c4ccdd3 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/meta/asynctask/CleanTaskTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core.store.meta.asynctask; + +import static org.junit.Assert.assertEquals; + +import org.apache.hugegraph.store.meta.asynctask.AbstractAsyncTask; +import org.apache.hugegraph.store.meta.asynctask.AsyncTask; +import org.apache.hugegraph.store.meta.asynctask.AsyncTaskState; +import org.apache.hugegraph.store.meta.asynctask.CleanTask; +import org.junit.Test; + +import org.apache.hugegraph.store.core.StoreEngineTestBase; + +public class CleanTaskTest extends StoreEngineTestBase { + + @Test + public void testSerialize() { + CleanTask task = new CleanTask(0, "graph0", AsyncTaskState.SUCCESS, null); + byte[] bytes = task.toBytes(); + + AsyncTask task2 = AbstractAsyncTask.fromBytes(bytes); + assertEquals(CleanTask.class, task2.getClass()); + System.out.println(task2); + + createPartitionEngine(0); + + CleanTask task3 = new CleanTask(0, "graph0", AsyncTaskState.START, null); + CleanTask task4 = new CleanTask(0, "graph0", AsyncTaskState.FAILED, null); + task3.handleTask(); + task4.handleTask(); + } + +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/util/MiscUtilClassTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/util/MiscUtilClassTest.java new file mode 100644 index 0000000000..0102dcd9c5 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/util/MiscUtilClassTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core.store.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.concurrent.TimeUnit; + +import org.apache.hugegraph.store.util.Asserts; +import org.apache.hugegraph.store.util.HgRaftError; +import org.apache.hugegraph.store.util.HgStoreException; +import org.apache.hugegraph.store.util.ManualResetEvent; +import org.junit.Test; + +import com.alipay.sofa.jraft.Status; + +public class MiscUtilClassTest { + + @Test + public void testHgRaftError() { + HgRaftError error = HgRaftError.forNumber(0); + assertEquals(0, error.getNumber()); + assertEquals("OK", error.getMsg()); + assertEquals(Status.OK().getCode(), error.toStatus().getCode()); + } + + @Test(expected = NullPointerException.class) + public void testAsserts() { + assertTrue(Asserts.isInvalid(null)); + assertTrue(Asserts.isInvalid()); + assertTrue(Asserts.isInvalid(null)); + assertTrue(Asserts.isInvalid("")); + assertFalse(Asserts.isInvalid("a")); + + Asserts.isNonNull(null); + Asserts.isNonNull(null, "msg"); + } + + @Test(expected = IllegalArgumentException.class) + public void testAsserts2() { + Asserts.isTrue(false, ""); + Asserts.isFalse(true, ""); + Asserts.isTrue(true, null); + } + + @Test + public void testHgStoreException() { + var exception = new HgStoreException(); + assertEquals(0, exception.getCode()); + exception = new HgStoreException("invalid"); + assertEquals(1000, exception.getCode()); + exception = new HgStoreException(1000, "invalid"); + assertEquals(1000, exception.getCode()); + exception = new HgStoreException(1000, new Throwable()); + assertEquals(1000, exception.getCode()); + exception = new HgStoreException("invalid", new Throwable()); + assertEquals(1000, exception.getCode()); + exception = new HgStoreException(1000, "%s", "invalid"); + assertEquals(1000, exception.getCode()); + } + + @Test + public void testManualResetEvent() throws InterruptedException { + ManualResetEvent event = new ManualResetEvent(false); + assertFalse(event.isSignalled()); + event.set(); + assertTrue(event.isSignalled()); + event.reset(); + assertFalse(event.waitOne(1, TimeUnit.SECONDS)); + event.set(); + event.waitOne(); + assertTrue(event.isSignalled()); + } + + +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/util/PartitionMetaStoreWrapperTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/util/PartitionMetaStoreWrapperTest.java new file mode 100644 index 0000000000..c15ddfeff0 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/util/PartitionMetaStoreWrapperTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core.store.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.store.core.StoreEngineTestBase; +import org.apache.hugegraph.store.meta.MetadataKeyHelper; +import org.apache.hugegraph.store.util.PartitionMetaStoreWrapper; +import org.junit.Before; +import org.junit.Test; + +public class PartitionMetaStoreWrapperTest extends StoreEngineTestBase { + + private PartitionMetaStoreWrapper wrapper; + + private Metapb.Partition partition; + + public static void putToDb(Metapb.Partition partition, PartitionMetaStoreWrapper wrapper) { + byte[] key = MetadataKeyHelper.getPartitionKey(partition.getGraphName(), partition.getId()); + wrapper.put(partition.getId(), key, partition.toByteArray()); + } + + @Before + public void setup() { + wrapper = new PartitionMetaStoreWrapper(); + partition = Metapb.Partition.newBuilder() + .setId(1) + .setGraphName("graph0") + .setStartKey(0L) + .setEndKey(65535L) + .build(); + } + + @Test + public void testGet() { + putToDb(partition, wrapper); + byte[] key = MetadataKeyHelper.getPartitionKey(partition.getGraphName(), partition.getId()); + assertEquals(partition, wrapper.get(1, key, Metapb.Partition.parser())); + byte[] key2 = MetadataKeyHelper.getPartitionKey("not_exists", partition.getId()); + assertNull(wrapper.get(1, key2, Metapb.Partition.parser())); + } + + @Test + public void testPut() { + putToDb(partition, wrapper); + byte[] key = MetadataKeyHelper.getPartitionKey(partition.getGraphName(), partition.getId()); + var list = wrapper.scan(partition.getId(), Metapb.Partition.parser(), key); + assertEquals(list.size(), 1); + assertEquals(list.get(0).getGraphName(), partition.getGraphName()); + } + + @Test + public void testDelete() { + putToDb(partition, wrapper); + byte[] key = MetadataKeyHelper.getPartitionKey(partition.getGraphName(), partition.getId()); + wrapper.delete(partition.getId(), key); + var list = wrapper.scan(partition.getId(), Metapb.Partition.parser(), key); + assertEquals(list.size(), 0); + } + + @Test + public void testScan() { + putToDb(partition, wrapper); + byte[] key = MetadataKeyHelper.getPartitionKey(partition.getGraphName(), partition.getId()); + var list = wrapper.scan(partition.getId(), Metapb.Partition.parser(), key); + assertEquals(list.size(), 1); + } + +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/util/ZipUtilsTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/util/ZipUtilsTest.java new file mode 100644 index 0000000000..1435c349ca --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/store/util/ZipUtilsTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core.store.util; + +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.zip.CRC32; + +import org.apache.hugegraph.rocksdb.access.util.ZipUtils; +import org.apache.hugegraph.store.UnitTestBase; +import org.apache.logging.log4j.core.util.FileUtils; +import org.junit.Before; +import org.junit.Test; + +public class ZipUtilsTest { + + private static final String ZIP_TEST_PATH = "/tmp/zip_util_test"; + + @Before + public void init() throws IOException { + UnitTestBase.deleteDir(new File(ZIP_TEST_PATH)); + FileUtils.mkdir(new File(ZIP_TEST_PATH), true); + FileUtils.mkdir(new File(ZIP_TEST_PATH + "/input"), true); + FileUtils.mkdir(new File(ZIP_TEST_PATH + "/output"), true); + Files.createFile(Paths.get(ZIP_TEST_PATH + "/input/foo.txt")); + } + + @Test + public void testZip() throws IOException { + ZipUtils.compress(ZIP_TEST_PATH, "input", ZIP_TEST_PATH + "/foo.zip", new CRC32()); + ZipUtils.decompress(ZIP_TEST_PATH + "/foo.zip", ZIP_TEST_PATH + "/output", new CRC32()); + assertTrue(Files.exists(Paths.get(ZIP_TEST_PATH + "/output/input/foo.txt"))); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/meta/GraphIDManagerTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/meta/GraphIDManagerTest.java new file mode 100644 index 0000000000..60bb542a9b --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/meta/GraphIDManagerTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.meta; + +import java.io.File; + +import org.apache.hugegraph.pd.common.PDException; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.store.UnitTestBase; +import org.apache.hugegraph.store.meta.base.DBSessionBuilder; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class GraphIDManagerTest extends UnitTestBase { + @Before + public void init() { + String dbPath = "/tmp/junit"; + UnitTestBase.deleteDir(new File(dbPath)); + super.initDB(dbPath); + } + + @Test + public void test() throws PDException { + GraphIdManager.maxGraphID = 64; + int max = GraphIdManager.maxGraphID; + try (RocksDBSession session = getDBSession("test")) { + GraphIdManager gid = new GraphIdManager(new DBSessionBuilder() { + @Override + public RocksDBSession getSession(int partId) { + return session.clone(); + } + }, 0); + for (int i = 0; i < max; i++) { + Assert.assertEquals(i, gid.getCId("Test", max)); + } + + Assert.assertEquals(-1, gid.getCId("Test", max)); + + gid.delCId("Test", 3); + Assert.assertEquals(3, gid.getCId("Test", max)); + Assert.assertEquals(-1, gid.getCId("Test", max)); + + long start = System.currentTimeMillis(); + for (int i = 0; i < GraphIdManager.maxGraphID; i++) { + long id = gid.getGraphId("g" + i); + Assert.assertEquals(i, id); + } + System.out.println("time is " + (System.currentTimeMillis() - start)); + { + gid.releaseGraphId("g" + 10); + long id = gid.getGraphId("g" + 10); + Assert.assertEquals(10, id); + } + start = System.currentTimeMillis(); + for (int i = 0; i < GraphIdManager.maxGraphID; i++) { + long id = gid.releaseGraphId("g" + i); + Assert.assertEquals(i, id); + } + System.out.println("time is " + (System.currentTimeMillis() - start)); + start = System.currentTimeMillis(); + for (int i = 0; i < GraphIdManager.maxGraphID; i++) { + long id = gid.getCId(GraphIdManager.GRAPH_ID_PREFIX, GraphIdManager.maxGraphID); + // long id = gid.getGraphId("g" + i); + Assert.assertTrue(id >= 0); + } + System.out.println("time is " + (System.currentTimeMillis() - start)); + } + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/meta/GraphManagerTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/meta/GraphManagerTest.java new file mode 100644 index 0000000000..8777700afe --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/meta/GraphManagerTest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.meta; + + +import org.junit.Assert; +// import org.junit.Test; + +public class GraphManagerTest { + // @Test + public void testCloneGraph() { + Graph graph = new Graph(); + graph.setGraphName("test1"); + + + Graph graph1 = graph.clone(); + + Assert.assertNotSame(graph, graph1); + + Assert.assertEquals(graph.getGraphName(), graph1.getGraphName()); + graph1.setGraphName("test4"); + + + Assert.assertNotEquals(graph.getGraphName(), graph1.getGraphName()); + + Assert.assertEquals(graph.getGraphName(), "test1"); + + } + +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/raftcore/BytesCarrierTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/raftcore/BytesCarrierTest.java new file mode 100644 index 0000000000..30142756c0 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/raftcore/BytesCarrierTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.raftcore; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.junit.Test; + +import com.google.protobuf.BytesCarrier; + +public class BytesCarrierTest { + + @Test + public void testWrite() throws IOException { + byte[] bytes = new byte[]{10, 20, 30}; + BytesCarrier carrier = new BytesCarrier(); + + // not valid + carrier.write((byte) 1); + assertNull(carrier.getValue()); + assertFalse(carrier.isValid()); + + // not valid + ByteBuffer buffer = ByteBuffer.allocate(10); + carrier.write(buffer); + assertNull(carrier.getValue()); + + // not valid + carrier.writeLazy(buffer); + assertNull(carrier.getValue()); + + // ok, write done + carrier.write(bytes, 0, bytes.length); + assertNotNull(carrier.getValue()); + assertTrue(carrier.isValid()); + + // has data + carrier.writeLazy(bytes, 0, bytes.length); + assertFalse(carrier.isValid()); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/raftcore/RaftSuiteTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/raftcore/RaftSuiteTest.java new file mode 100644 index 0000000000..f3b1f31d29 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/raftcore/RaftSuiteTest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.raftcore; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +@RunWith(Suite.class) +@Suite.SuiteClasses({ + BytesCarrierTest.class, + ZeroByteStringHelperTest.class +}) +public class RaftSuiteTest { + +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/raftcore/ZeroByteStringHelperTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/raftcore/ZeroByteStringHelperTest.java new file mode 100644 index 0000000000..9d702e3ff7 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/raftcore/ZeroByteStringHelperTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.raftcore; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; + +import org.junit.Test; + +import com.google.protobuf.ZeroByteStringHelper; + +public class ZeroByteStringHelperTest { + + private static final String STR = "hello word!"; + + @Test + public void testWrap() { + byte[] b1 = new byte[]{10, 20, 30}; + byte[] b2 = new byte[]{40, 50}; + + var h1 = ZeroByteStringHelper.wrap(b1); + var h2 = ZeroByteStringHelper.wrap(b2, 0, b2.length); + + ByteBuffer buffer = ByteBuffer.allocate(5); + buffer.put(b1); + buffer.put(b2); + var h3 = ZeroByteStringHelper.wrap(buffer); + assertEquals(h3.isEmpty(), true); + var h4 = ZeroByteStringHelper.concatenate(h1, h2); + assertTrue(Arrays.equals(ZeroByteStringHelper.getByteArray(h4), buffer.array())); + } + + @Test + public void testConcatenate() { + byte[] b1 = new byte[]{10, 20, 30}; + byte[] b2 = new byte[]{40, 50}; + ByteBuffer buffer1 = ByteBuffer.allocate(5); + buffer1.put(b1); + + ByteBuffer buffer2 = ByteBuffer.allocate(5); + buffer1.put(b2); + + var array = new ArrayList(); + array.add(buffer1); + array.add(buffer2); + + var bs = ZeroByteStringHelper.concatenate(array); + assertEquals(bs.toByteArray().length, 5); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/rocksdb/BaseRocksDbTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/rocksdb/BaseRocksDbTest.java new file mode 100644 index 0000000000..4fd36665e9 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/rocksdb/BaseRocksDbTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.rocksdb; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.config.OptionSpace; +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBOptions; +import org.junit.After; +import org.junit.BeforeClass; + +public class BaseRocksDbTest { + @BeforeClass + public static void init() { + OptionSpace.register("org/apache/hugegraph/store/rocksdb", + "org.apache.hugegraph.rocksdb.access.RocksDBOptions"); + RocksDBOptions.instance(); + + Map configMap = new HashMap<>(); + configMap.put("rocksdb.write_buffer_size", "1048576"); + configMap.put("rocksdb.bloom_filter_bits_per_key", "10"); + + HugeConfig hConfig = new HugeConfig(configMap); + RocksDBFactory rFactory = RocksDBFactory.getInstance(); + rFactory.setHugeConfig(hConfig); + + } + + @After + public void teardown() throws Exception { + // pass + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/rocksdb/RocksDBFactoryTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/rocksdb/RocksDBFactoryTest.java new file mode 100644 index 0000000000..61931fac9c --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/rocksdb/RocksDBFactoryTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.rocksdb; + +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.SessionOperator; +import org.junit.Test; + +public class RocksDBFactoryTest extends BaseRocksDbTest { + @Test + public void testCreateSession() { + RocksDBFactory factory = RocksDBFactory.getInstance(); + try (RocksDBSession dbSession = factory.createGraphDB("./tmp", "test1")) { + SessionOperator op = dbSession.sessionOp(); + op.prepare(); + try { + op.put("tbl", "k1".getBytes(), "v1".getBytes()); + op.commit(); + } catch (Exception e) { + op.rollback(); + } + + } + factory.destroyGraphDB("test1"); + } + + @Test + public void testTotalKeys() { + RocksDBFactory dbFactory = RocksDBFactory.getInstance(); + System.out.println(dbFactory.getTotalSize()); + + System.out.println(dbFactory.getTotalKey().entrySet() + .stream().map(e -> e.getValue()).reduce(0L, Long::sum)); + } + + @Test + public void releaseAllGraphDB() { + System.out.println(RocksDBFactory.class); + + RocksDBFactory rFactory = RocksDBFactory.getInstance(); + + if (rFactory.queryGraphDB("bj01") == null) { + rFactory.createGraphDB("./tmp", "bj01"); + } + + if (rFactory.queryGraphDB("bj02") == null) { + rFactory.createGraphDB("./tmp", "bj02"); + } + + if (rFactory.queryGraphDB("bj03") == null) { + rFactory.createGraphDB("./tmp", "bj03"); + } + + RocksDBSession dbSession = rFactory.queryGraphDB("bj01"); + + dbSession.checkTable("test"); + SessionOperator sessionOp = dbSession.sessionOp(); + sessionOp.prepare(); + + sessionOp.put("test", "hi".getBytes(), "byebye".getBytes()); + sessionOp.commit(); + + rFactory.releaseAllGraphDB(); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/rocksdb/RocksDbSuiteTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/rocksdb/RocksDbSuiteTest.java new file mode 100644 index 0000000000..59c1cd9986 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/rocksdb/RocksDbSuiteTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.rocksdb; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +import lombok.extern.slf4j.Slf4j; + +@RunWith(Suite.class) +@Suite.SuiteClasses({ + RocksDBFactoryTest.class +}) + +@Slf4j +public class RocksDbSuiteTest { +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/service/BaseServerTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/service/BaseServerTest.java new file mode 100644 index 0000000000..ca0a8fd248 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/service/BaseServerTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.service; + +import org.junit.After; +import org.junit.BeforeClass; + +public class BaseServerTest { + @BeforeClass + public static void init() { + + } + + @After + public void teardown() { + // pass + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/service/ServerSuiteTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/service/ServerSuiteTest.java new file mode 100644 index 0000000000..00a58d490e --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/service/ServerSuiteTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.service; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +import lombok.extern.slf4j.Slf4j; + +@RunWith(Suite.class) +@Suite.SuiteClasses({ +}) + +@Slf4j + +public class ServerSuiteTest { +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/service/SessionTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/service/SessionTest.java new file mode 100644 index 0000000000..38449a106e --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/service/SessionTest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.service; + +import org.junit.Test; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class SessionTest { + @Test + public void testQuota() { + + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/util/CopyOnWriteCacheTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/util/CopyOnWriteCacheTest.java new file mode 100644 index 0000000000..86d03fd534 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/util/CopyOnWriteCacheTest.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.util; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.caucho.hessian.io.Hessian2Input; +import com.caucho.hessian.io.Hessian2Output; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Map; +import java.util.Set; + +import org.apache.hugegraph.store.util.CopyOnWriteCache; +import org.junit.Before; +import org.junit.Test; + +public class CopyOnWriteCacheTest { + + private CopyOnWriteCache writeCache; + + @Before + public void setUp() { + writeCache = new CopyOnWriteCache<>(5000L); + } + + @Test + public void testContainsKey() { + // Setup + // Run the test + writeCache.put("k", "v"); + final boolean result = writeCache.containsKey("k"); + Map allKeys = + Map.ofEntries(Map.entry("key1", "value1")); + writeCache.putAll(allKeys); + // Verify the results + assertTrue(result); + final Set> entries = writeCache.entrySet(); + Set keySet = writeCache.keySet(); + String val = writeCache.get("k"); + boolean isEmpty = writeCache.isEmpty(); + writeCache.size(); + writeCache.values(); + // Verify the results + assertFalse(isEmpty); + writeCache.remove("k"); + writeCache.putIfAbsent("k", "v"); + writeCache.replace("k", "original", "replacement"); + writeCache.replace("k", "v"); + writeCache.clear(); + assertTrue(writeCache.isEmpty()); + } + + // @Test + public void testCache() throws InterruptedException { + Map cache = new CopyOnWriteCache<>(1000); + cache.put("1", "1"); + Thread.sleep(2000); + Asserts.isTrue(!cache.containsKey("1"), "cache do not clear"); + } + + // @Test + public void test() { + + byte[] bytes; + try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) { + long[] l = new long[]{1, 2}; + Hessian2Output output = new Hessian2Output(bos); + output.writeObject(l); + output.flush(); + bytes = bos.toByteArray(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + try (ByteArrayInputStream bis = new ByteArrayInputStream(bytes)) { + Hessian2Input input = new Hessian2Input(bis); + long[] obj = (long[]) input.readObject(); + input.close(); + + for (long l : obj) { + System.out.println(l); + } + + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/util/FutureClosureTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/util/FutureClosureTest.java new file mode 100644 index 0000000000..1c06f97f26 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/util/FutureClosureTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.util; + +import org.junit.Assert; +import org.junit.Test; + +import com.alipay.sofa.jraft.Status; + +public class FutureClosureTest { + @Test + public void test() { + FutureClosure closure = new FutureClosure(); + new Thread(() -> { + try { + Thread.sleep(1000); + closure.run(Status.OK()); + } catch (InterruptedException e) { + closure.run(new Status(-1, e.getMessage())); + } + + }).start(); + + Assert.assertEquals(closure.get().getCode(), Status.OK().getCode()); + Assert.assertEquals(closure.get().getCode(), Status.OK().getCode()); + Assert.assertEquals(closure.get().getCode(), Status.OK().getCode()); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/util/HgStoreTestUtil.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/util/HgStoreTestUtil.java new file mode 100644 index 0000000000..6a19a026b6 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/util/HgStoreTestUtil.java @@ -0,0 +1,486 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.util; + +import java.io.Closeable; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; + +import org.apache.hugegraph.store.HgKvEntry; +import org.apache.hugegraph.store.HgKvIterator; +import org.apache.hugegraph.store.HgOwnerKey; +import org.apache.hugegraph.store.HgStoreSession; +import org.apache.hugegraph.store.client.util.HgStoreClientConst; +import org.apache.hugegraph.store.client.util.MetricX; + +public class HgStoreTestUtil { + public static final String GRAPH_NAME = "default/hugegraph/g"; + public static final String GRAPH_NAME2 = "default/hugegraph2/g"; + public static final String TABLE_NAME = "unit-table"; + public static final String TABLE_NAME2 = "unit-table-2"; + + public static Map batchPut(HgStoreSession session, String keyPrefix) { + return batchPut(session, keyPrefix, 100); + } + + public static Map batchPut(HgStoreSession session, String keyPrefix, + int loop) { + return batchPut(session, TABLE_NAME, keyPrefix, loop); + } + + public static Map batchPut( + HgStoreSession session, String tableName, String keyPrefix, int loop) { + return batchPut(session, tableName, keyPrefix, loop, 1, key -> toOwnerKey(key)); + } + + public static Map batchPut( + HgStoreSession session, String tableName, byte[] keyPrefix, int loop) { + return batchPut(session, tableName, keyPrefix, loop, 1, + (prefix, key) -> toOwnerKey(prefix, key)); + } + + public static Map batchPut( + HgStoreSession session, String tableName, String keyPrefix, int loop, int start) { + return batchPut(session, tableName, keyPrefix, loop, start, key -> toOwnerKey(key)); + } + + public static Map batchPut( + HgStoreSession session, String tableName, String keyPrefix, int loop, + Function f) { + return batchPut(session, tableName, keyPrefix, loop, 1, f); + } + + public static Map batchPut( + HgStoreSession session, + String tableName, + String keyPrefix, + int loop, + int start, + Function f) { + + Map res = new LinkedHashMap<>(); + + int length = String.valueOf(loop).length(); + + session.beginTx(); + for (int i = start; i <= loop; i++) { + + HgOwnerKey key = f.apply(keyPrefix + "-" + padLeftZeros(String.valueOf(i), length)); + + byte[] value = toBytes(keyPrefix + "-V-" + i); + res.put(key, value); + session.put(tableName, key, value); + + if ((i + 1) % 10000 == 0) { + println("commit: " + (i + 1)); + session.commit(); + session.beginTx(); + } + } + if (session.isTx()) { + session.commit(); + } + + return res; + } + + public static Map batchPut( + HgStoreSession session, + String tableName, + byte[] keyPrefix, + int loop, + int start, + BiFunction f) { + + Map res = new LinkedHashMap<>(); + + int length = String.valueOf(loop).length(); + + session.beginTx(); + for (int i = start; i <= loop; i++) { + + HgOwnerKey key = f.apply(keyPrefix, padLeftZeros(String.valueOf(i), length)); + + byte[] value = toBytes(keyPrefix + "-V-" + i); + res.put(key, value); + session.put(tableName, key, value); + + if ((i + 1) % 10000 == 0) { + println("commit: " + (i + 1)); + session.commit(); + session.beginTx(); + } + } + if (session.isTx()) { + session.commit(); + } + + return res; + } + + /*---- common -----*/ + public static void printOwner(List list) { + if (list == null) { + return; + } + + for (HgOwnerKey entry : list) { + println(entry); + } + } + + public static void printNum(List list, String title) { + if (list == null) { + return; + } + + println(title + " size: " + list.size()); + } + + public static int println(Iterable> iterators) { + AtomicInteger counter = new AtomicInteger(); + iterators.forEach(e -> counter.addAndGet(HgStoreTestUtil.println(e))); + return counter.get(); + } + + public static int println(HgKvIterator iterator) { + if (iterator == null) { + return 0; + } + + AtomicInteger counter = new AtomicInteger(); + + while (iterator.hasNext()) { + counter.incrementAndGet(); + print(iterator.next()); + println(" " + Arrays.toString(iterator.position())); + } + + iterator.close(); + + return counter.get(); + } + + public static void println(HgKvIterator iterator, Function mod) { + if (iterator == null) { + return; + } + int count = 0; + + while (iterator.hasNext()) { + count++; + if (mod.apply(count)) { + print(iterator.next()); + println(" " + Arrays.toString(iterator.position())); + } + } + + iterator.close(); + } + + public static void println(List list) { + if (list == null) { + return; + } + + for (HgKvEntry entry : list) { + println(entry); + } + } + + public static void println(List list, int mod) { + if (list == null) { + return; + } + + for (int i = 0; i < list.size(); i++) { + if (i % mod == 0) { + println(list.get(i)); + } + } + } + + public static void println(HgKvEntry kv) { + if (kv == null) { + System.out.println("null"); + return; + } + println("[ " + kv.code() + " " + toStr(kv.key()) + " : " + toStr(kv.value()) + " ]"); + } + + public static void println(HgOwnerKey key) { + if (key == null) { + System.out.println("null"); + return; + } + println("[ " + toInt(key.getOwner()) + " : " + toStr(key.getKey()) + " ]"); + } + + public static void println(String str) { + System.out.println(str); + } + + public static void println(Number num) { + System.out.println(num); + } + + public static void print(String str) { + System.out.print(str); + } + + public static void print(HgKvEntry kv) { + if (kv == null) { + System.out.println("null"); + return; + } + print("[ " + kv.code() + " " + toStr(kv.key()) + " : " + toStr(kv.value()) + " ]"); + } + + private static byte[] getOwner(String key) { + return getOwner(toBytes(key)); + } + + private static byte[] getOwner(byte[] key) { + return toBytes(Arrays.hashCode(key)); + } + + public static HgOwnerKey toAllPartitionKey(String key) { + return HgOwnerKey.of(HgStoreClientConst.ALL_PARTITION_OWNER, toBytes(key)); + } + + public static HgOwnerKey toAllPartitionKey(byte[] prefix, String key) { + return HgOwnerKey.of(HgStoreClientConst.ALL_PARTITION_OWNER, toBytes(prefix, key)); + } + + public static HgOwnerKey toAllPartitionKey(byte[] prefix) { + return HgOwnerKey.of(HgStoreClientConst.ALL_PARTITION_OWNER, prefix); + } + + public static HgOwnerKey toOwnerKey(byte[] prefix, String key) { + byte[] newKey = toBytes(prefix, key); + return new HgOwnerKey(getOwner(newKey), newKey); + } + + public static HgOwnerKey toOwnerKey(String key) { + return new HgOwnerKey(getOwner(key), toBytes(key)); + } + + public static HgOwnerKey toOwnerKey(byte[] key) { + return new HgOwnerKey(getOwner(key), key); + } + + public static HgOwnerKey toOwnerKey(String owner, String key) { + return HgOwnerKey.of(toBytes(owner), toBytes(key)); + } + + public static HgOwnerKey toOwnerKey(int code, String key) { + return HgOwnerKey.of(code, toBytes(key)); + } + + public static String toStr(byte[] b) { + if (b == null) { + return ""; + } + if (b.length == 0) { + return ""; + } + return new String(b, StandardCharsets.UTF_8); + } + + public static byte[] toBytes(byte[] prefix, String str) { + if (str == null) { + return null; + } + byte[] buf = str.getBytes(StandardCharsets.UTF_8); + byte[] res = new byte[buf.length + prefix.length]; + System.arraycopy(prefix, 0, res, 0, prefix.length); + System.arraycopy(buf, 0, res, prefix.length, buf.length); + return res; + } + + public static byte[] toBytes(String str) { + if (str == null) { + return null; + } + return str.getBytes(StandardCharsets.UTF_8); + } + + public static byte[] toBytes(long l) { + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); + buffer.putLong(l); + return buffer.array(); + } + + private static byte[] toBytes(final int i) { + ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES); + buffer.putInt(i); + return buffer.array(); + } + + public static long toLong(byte[] bytes) { + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); + buffer.put(bytes); + buffer.flip(); // need flip + return buffer.getLong(); + } + + public static int toInt(byte[] bytes) { + ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES); + buffer.put(bytes); + buffer.flip(); // need flip + return buffer.getInt(); + } + + public static String padLeftZeros(String str, int n) { + return String.format("%1$" + n + "s", str).replace(' ', '0'); + } + + public static String toSuffix(int num, int length) { + return "-" + padLeftZeros(String.valueOf(num), length); + } + + public static int amountOf(List list) { + if (list == null) { + return 0; + } + return list.size(); + } + + public static int amountOf(Iterator iterator) { + if (iterator == null) { + return 0; + } + int count = 0; + while (iterator.hasNext()) { + Object ignore = iterator.next(); + ++count; + } + return count; + } + + public static int oOMAmountOf(Iterator iterator) { + if (iterator == null) { + return 0; + } + int count = 0; + while (iterator.hasNext()) { + Object obj = iterator.next(); + ++count; + if (count % 10000 == 0) { + println(count); + sleeping(10); + } + if (count % 100000 == 0) { + break; + } + } + if (iterator instanceof Closeable) { + try { + ((Closeable) iterator).close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + return count; + } + + public static int amountIn(List iterators) { + return iterators.stream().map(e -> HgStoreTestUtil.amountOf(e)).reduce(0, Integer::sum); + } + + public static void sleeping(long time) { + try { + Thread.sleep(time); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + public static void parallelTest(int threads, Runnable runner, + Consumer throwableConsumer) { + int threadsAmount = threads; + CountDownLatch countDownLatch = new CountDownLatch(threadsAmount); + ExecutorService pool = new ThreadPoolExecutor(threadsAmount, threadsAmount + 20, + 200, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(1000)); + for (int i = 0; i < threadsAmount; i++) { + pool.submit( + () -> { + try { + runner.run(); + } catch (Throwable t) { + throwableConsumer.accept(t); + } + countDownLatch.countDown(); + }); + } + + try { + countDownLatch.await(); + pool.shutdown(); + + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + public static void repeatedlyTest(int times, Runnable runner) { + MetricX metrics = MetricX.ofStart(); + + for (int j = 0; j < times; j++) { + try { + runner.run(); + Thread.sleep(1); + } catch (InterruptedException e) { + e.printStackTrace(); + break; + } catch (Throwable t) { + metrics.countFail(); + t.printStackTrace(); + } + } + metrics.end(); + System.out.println("*************************************************"); + System.out.println("************* Repeatedly Test Completed **************"); + System.out.println("Total: " + metrics.past() / 1000 + " sec."); + System.out.println("Repeated: " + times + " times."); + System.out.println("Fail: " + metrics.getFailureCount() + " times."); + System.out.println("*************************************************"); + } + + public static void runWaiting() { + while (true) { + sleeping(1000); + } + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/util/UnsafeUtilTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/util/UnsafeUtilTest.java new file mode 100644 index 0000000000..5e3c40c017 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/util/UnsafeUtilTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.lang.reflect.Field; + +import org.apache.hugegraph.store.util.UnsafeUtf8Util; +import org.apache.hugegraph.store.util.UnsafeUtil; +import org.junit.Test; + +import lombok.Data; +import sun.misc.Unsafe; + +public class UnsafeUtilTest { + + + @Test + public void testHasUnsafe() { + assertTrue(UnsafeUtil.hasUnsafe()); + } + + @Test + public void testPutByte() { + UnsafeUtil.putByte("content".getBytes(), 0L, (byte) 99); + assertEquals((byte) 99, UnsafeUtil.getByte("content".getBytes(), 0L)); + } + + @Test + public void testMoveToString() { + assertEquals("a", UnsafeUtil.moveToString(new char[]{'a'})); + } + + @Test + public void testEncodedLength() { + assertEquals(10, UnsafeUtf8Util.encodedLength("aa中文aa")); + } + + @Test + public void testEncodeUtf8() { + assertEquals(10, UnsafeUtf8Util.encodeUtf8("aa中文aa", new byte[16], 0, 16)); + } + + @Test + public void testDecodeUtf8() { + assertEquals("co", UnsafeUtf8Util.decodeUtf8("content".getBytes(), 0, 2)); + } + + @Test + public void testUnsafeUtf8Util() { + String content = "content"; + UnsafeUtf8Util.decodeUtf8(content.getBytes(), 0, content.length()); + byte[] out = new byte[content.length()]; + UnsafeUtf8Util.encodeUtf8(content, out, 0, content.length()); + UnsafeUtf8Util.encodedLength(content); + } + + @Test + public void testUnsafeAccessor() { + Unsafe unsafe = null; + long offset = 0; + try { + Field unsafeField = Unsafe.class.getDeclaredField("theUnsafe"); + unsafeField.setAccessible(true); + unsafe = (Unsafe) unsafeField.get(null); + UnsafeUtil.UnsafeAccessor acc = new UnsafeUtil.UnsafeAccessor(unsafe); + acc.getUnsafe(); + TestObject to = new TestObject(); + byte byteValue = 126; + offset = acc.objectFieldOffset(TestObject.class.getDeclaredField("b")); + acc.putByte(to, offset, byteValue); + byte b = acc.getByte(to, offset); + assertEquals(byteValue, b); + short shortValue = 1; + acc.putShort(to, offset, shortValue); + short shortResult = acc.getShort(to, offset); + assertEquals(shortValue, shortResult); + int intValue = 99; + acc.putInt(to, offset, intValue); + int i = acc.getInt(to, offset); + assertEquals(intValue, i); + long longValue = 11L; + acc.putLong(to, offset, longValue); + long l = acc.getLong(to, offset); + assertEquals(longValue, l); + acc.putBoolean(to, offset, false); + assertFalse(acc.getBoolean(to, offset)); + float f = 1; + acc.putFloat(to, offset, f); + float v = acc.getFloat(to, offset); + assertEquals(f, v, 0.0); + double d = 2; + acc.putDouble(to, offset, d); + double v1 = acc.getDouble(to, offset); + assertEquals(d, v1, 0.0); + TestObject o1 = new TestObject(); + acc.putObject(to, offset, o1); + Object o2 = acc.getObject(to, offset); + assertEquals(o1, o2); + offset = unsafe.allocateMemory(1024); + acc.putByte(offset, byteValue); + byte bResult = acc.getByte(offset); + assertEquals(byteValue, bResult); + acc.putShort(offset, shortValue); + short aShort1 = acc.getShort(offset); + assertEquals(aShort1, shortValue); + acc.putInt(offset, intValue); + assertEquals(acc.getInt(offset), intValue); + acc.putLong(offset, longValue); + assertEquals(acc.getLong(offset), longValue); + unsafe.freeMemory(offset); + offset = acc.objectFieldOffset(TestObject.class.getDeclaredField("b")); + acc.putByteVolatile(to, offset, byteValue); + b = acc.getByteVolatile(to, offset); + assertEquals(byteValue, b); + acc.putShortVolatile(to, offset, shortValue); + shortResult = acc.getShortVolatile(to, offset); + assertEquals(shortValue, shortResult); + acc.putIntVolatile(to, offset, intValue); + i = acc.getIntVolatile(to, offset); + assertEquals(intValue, i); + acc.putLongVolatile(to, offset, longValue); + l = acc.getLongVolatile(to, offset); + assertEquals(longValue, l); + acc.putBooleanVolatile(to, offset, false); + assertFalse(acc.getBooleanVolatile(to, offset)); + acc.putFloatVolatile(to, offset, f); + v = acc.getFloatVolatile(to, offset); + assertEquals(f, v, 0.0); + acc.putDoubleVolatile(to, offset, d); + v1 = acc.getDoubleVolatile(to, offset); + assertEquals(d, v1, 0.0); + acc.putObjectVolatile(to, offset, o1); + o2 = acc.getObjectVolatile(to, offset); + assertEquals(o1, o2); + + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Data + private class TestObject { + private Object o; + private byte b; + + public TestObject() { + + } + + public TestObject(Object o, byte b) { + this.o = o; + this.b = b; + } + } + +} diff --git a/hugegraph-store/hg-store-test/src/main/resources/log4j2.xml b/hugegraph-store/hg-store-test/src/main/resources/log4j2.xml index 0a3aae7b9d..c8fa4195f6 100644 --- a/hugegraph-store/hg-store-test/src/main/resources/log4j2.xml +++ b/hugegraph-store/hg-store-test/src/main/resources/log4j2.xml @@ -26,8 +26,8 @@ - - + + @@ -37,24 +37,24 @@ - - + + - - + + - + - - + + @@ -64,25 +64,25 @@ - + - + - - + + - - + + - - + + @@ -91,11 +91,11 @@ - + - - + + diff --git a/hugegraph-store/pom.xml b/hugegraph-store/pom.xml index be1bbfc7ff..0174de1aed 100644 --- a/hugegraph-store/pom.xml +++ b/hugegraph-store/pom.xml @@ -19,9 +19,7 @@ - 4.0.0 - hugegraph-store ${revision} pom @@ -77,12 +75,11 @@ hg-store-core ${project.version} - - - - - - + + org.apache.hugegraph + hg-store-transfer + ${project.version} + org.apache.logging.log4j log4j-slf4j-impl @@ -155,56 +152,55 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + store-client-test + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.20 + + + store-client-test + + test + + test + + + + + + + + store-core-test + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.20 + + + store-core-test + + test + + test + + + + + + store-common-test @@ -229,77 +225,77 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + store-rocksdb-test + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.20 + + + sit -test + + test + + test + + + + + + + + store-server-test + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.20 + + + store-server-test + + test + + test + + + + + + + + store-raftcore-test + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.20 + + + store-raftcore-test + + test + + test + + + + + +