diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/profile/GraphsAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/profile/GraphsAPI.java index f1793f1f64..b09ac224e7 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/profile/GraphsAPI.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/profile/GraphsAPI.java @@ -40,6 +40,7 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.SecurityContext; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import com.baidu.hugegraph.HugeGraph; @@ -127,16 +128,12 @@ public Object create(@Context GraphManager manager, @PathParam("name") String name, @QueryParam("clone_graph_name") String clone, String configText) { - E.checkArgument(configText != null && !configText.isEmpty(), - "The config text can't be null or empty"); + LOG.debug("Create graph '{}' with clone graph '{}', config text '{}'", + name, clone, configText); HugeGraph graph; - if (clone != null && !clone.isEmpty()) { - LOG.debug("Create graph {} with copied config from '{}'", - name, clone); + if (StringUtils.isNotEmpty(clone)) { graph = manager.cloneGraph(clone, name, configText); } else { - LOG.debug("Create graph {} with config options '{}'", name, - configText); graph = manager.createGraph(name, configText); } return ImmutableMap.of("name", graph.name(), diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/ContextGremlinServer.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/ContextGremlinServer.java index a7ce3e4ff2..144a7e9543 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/ContextGremlinServer.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/ContextGremlinServer.java @@ -62,17 +62,17 @@ public ContextGremlinServer(final Settings settings, EventHub eventHub) { private void listenChanges() { this.eventHub.listen(Events.GRAPH_CREATE, event -> { - LOG.debug("GremlinServer accepts event 'graph.create'"); + LOG.debug("GremlinServer accepts event '{}'", event.name()); event.checkArgs(HugeGraph.class); HugeGraph graph = (HugeGraph) event.args()[0]; this.injectGraph(graph); return null; }); this.eventHub.listen(Events.GRAPH_DROP, event -> { - LOG.debug("GremlinServer accepts event 'graph.drop'"); - event.checkArgs(String.class); - String name = (String) event.args()[0]; - this.removeGraph(name); + LOG.debug("GremlinServer accepts event '{}'", event.name()); + event.checkArgs(HugeGraph.class); + HugeGraph graph = (HugeGraph) event.args()[0]; + this.removeGraph(graph.name()); return null; }); } diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java index 359817a963..31bcf7960a 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java @@ -739,6 +739,12 @@ public void resumeSnapshot() { this.hugegraph.resumeSnapshot(); } + @Override + public void create(String configPath, Id server, NodeRole role) { + this.verifyPermission(HugePermission.WRITE, ResourceType.STATUS); + this.hugegraph.create(configPath, server, role); + } + @Override public void drop() { this.verifyPermission(HugePermission.WRITE, ResourceType.STATUS); @@ -746,9 +752,9 @@ public void drop() { } @Override - public HugeConfig cloneConfig() { + public HugeConfig cloneConfig(String newGraph) { this.verifyPermission(HugePermission.WRITE, ResourceType.STATUS); - return this.hugegraph.cloneConfig(); + return this.hugegraph.cloneConfig(newGraph); } private Cache cache(String prefix, long capacity, diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java index 94887b07f9..be36b1af4d 100644 --- a/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java +++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/core/GraphManager.java @@ -24,10 +24,12 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.commons.lang3.StringUtils; import org.apache.tinkerpop.gremlin.server.auth.AuthenticationException; import org.apache.tinkerpop.gremlin.server.util.MetricManager; import org.apache.tinkerpop.gremlin.structure.Graph; @@ -64,7 +66,6 @@ import com.baidu.hugegraph.serializer.Serializer; import com.baidu.hugegraph.server.RestServer; import com.baidu.hugegraph.task.TaskManager; -import com.baidu.hugegraph.type.define.GraphMode; import com.baidu.hugegraph.type.define.NodeRole; import com.baidu.hugegraph.util.ConfigUtil; import com.baidu.hugegraph.util.E; @@ -133,38 +134,39 @@ public HugeGraph cloneGraph(String name, String newName, * 3. inject graph and traversal source into gremlin server context * 4. inject graph into rest server context */ - HugeGraph g = this.graph(name); - E.checkArgumentNotNull(g, "The origin graph '%s' doesn't exist", name); - E.checkArgumentNotNull(newName, "The graph name can't be null"); + HugeGraph cloneGraph = this.graph(name); + E.checkArgumentNotNull(cloneGraph, + "The clone graph '%s' doesn't exist", name); + E.checkArgument(StringUtils.isNotEmpty(newName), + "The graph name can't be null or empty"); E.checkArgument(!this.graphs().contains(newName), "The graph '%s' has existed", newName); - PropertiesConfiguration propConfig = ConfigUtil.buildConfig(configText); - HugeConfig cloneConfig = g.cloneConfig(); - cloneConfig.setDelimiterParsingDisabled(true); - // Use the passed config to overwrite the old one - propConfig.getKeys().forEachRemaining(key -> { - cloneConfig.setProperty(key, propConfig.getProperty(key)); - }); - this.checkOptions(cloneConfig); - HugeGraph graph = this.createGraph(cloneConfig); - // Write config to disk file - ConfigUtil.writeToFile(this.graphsDir, graph.name(), cloneConfig); - return graph; + HugeConfig cloneConfig = cloneGraph.cloneConfig(newName); + if (StringUtils.isNotEmpty(configText)) { + PropertiesConfiguration propConfig = ConfigUtil.buildConfig( + configText); + // Use the passed config to overwrite the old one + propConfig.getKeys().forEachRemaining(key -> { + cloneConfig.setProperty(key, propConfig.getProperty(key)); + }); + this.checkOptions(cloneConfig); + } + + return this.createGraph(cloneConfig, newName); } public HugeGraph createGraph(String name, String configText) { - E.checkArgumentNotNull(name, "The graph name can't be null"); + E.checkArgument(StringUtils.isNotEmpty(name), + "The graph name can't be null or empty"); E.checkArgument(!this.graphs().contains(name), "The graph name '%s' has existed", name); PropertiesConfiguration propConfig = ConfigUtil.buildConfig(configText); HugeConfig config = new HugeConfig(propConfig); this.checkOptions(config); - HugeGraph graph = this.createGraph(config); - // Write config to disk file - ConfigUtil.writeToFile(this.graphsDir, graph.name(), config); - return graph; + + return this.createGraph(config, name); } public void dropGraph(String name) { @@ -173,9 +175,11 @@ public void dropGraph(String name) { E.checkArgument(this.graphs.size() > 1, "The graph '%s' is the only one, not allowed to delete", name); - graph.drop(); + + this.dropGraph(graph); + // Let gremlin server and rest server context remove graph - this.eventHub.notify(Events.GRAPH_DROP, name); + this.notifyAndWaitEvent(Events.GRAPH_DROP, graph); } public Set graphs() { @@ -362,9 +366,9 @@ private void checkBackendVersionOrExit(HugeConfig config) { private void serverStarted(HugeConfig config) { String server = config.get(ServerOptions.SERVER_ID); String role = config.get(ServerOptions.SERVER_ROLE); - E.checkArgument(server != null && !server.isEmpty(), + E.checkArgument(StringUtils.isNotEmpty(server), "The server name can't be null or empty"); - E.checkArgument(role != null && !role.isEmpty(), + E.checkArgument(StringUtils.isNotEmpty(role), "The server role can't be null or empty"); this.server = IdGenerator.of(server); this.role = NodeRole.valueOf(role.toUpperCase()); @@ -414,17 +418,17 @@ private void addMetrics(HugeConfig config) { private void listenChanges() { this.eventHub.listen(Events.GRAPH_CREATE, event -> { - LOG.debug("RestServer accepts event {}", event.name()); + LOG.debug("RestServer accepts event '{}'", event.name()); event.checkArgs(HugeGraph.class); HugeGraph graph = (HugeGraph) event.args()[0]; this.graphs.put(graph.name(), graph); return null; }); this.eventHub.listen(Events.GRAPH_DROP, event -> { - LOG.debug("RestServer accepts event {}", event.name()); - event.checkArgs(String.class); - String name = (String) event.args()[0]; - this.graphs.remove(name); + LOG.debug("RestServer accepts event '{}'", event.name()); + event.checkArgs(HugeGraph.class); + HugeGraph graph = (HugeGraph) event.args()[0]; + this.graphs.remove(graph.name()); return null; }); } @@ -434,45 +438,60 @@ private void unlistenChanges() { this.eventHub.unlisten(Events.GRAPH_DROP); } - private HugeGraph createGraph(HugeConfig config) { - // Will fill graph instance into HugeFactory.graphs after open succeed - HugeGraph graph = (HugeGraph) GraphFactory.open(config); - if (this.requireAuthentication()) { - /* - * The main purpose is to call method - * verifyPermission(HugePermission.WRITE, ResourceType.STATUS) - * that is private - */ - graph.mode(GraphMode.NONE); + private void notifyAndWaitEvent(String event, HugeGraph graph) { + Future future = this.eventHub.notify(event, graph); + try { + future.get(); + } catch (Throwable e) { + LOG.warn("Error when waiting for event execution: {}", event, e); } + } + private HugeGraph createGraph(HugeConfig config, String name) { + HugeGraph graph = null; try { - graph.initBackend(); - graph.serverStarted(this.server, this.role); - } catch (BackendException e) { - HugeFactory.remove(graph); + // Create graph instance + graph = (HugeGraph) GraphFactory.open(config); + + // Init graph and start it + graph.create(this.graphsDir, this.server, this.role); + } catch (Throwable e) { + LOG.error("Failed to create graph '{}' due to: {}", + name, e.getMessage(), e); + if (graph != null) { + this.dropGraph(graph); + } throw e; } + // Let gremlin server and rest server add graph to context - this.eventHub.notify(Events.GRAPH_CREATE, graph); + this.notifyAndWaitEvent(Events.GRAPH_CREATE, graph); + return graph; } + private void dropGraph(HugeGraph graph) { + // Clear data and config files + graph.drop(); + + /* + * Will fill graph instance into HugeFactory.graphs after + * GraphFactory.open() succeed, remove it when graph drop + */ + HugeFactory.remove(graph); + } + private void checkOptions(HugeConfig config) { // The store cannot be the same as the existing graph - this.checkOptionsUnique(config, CoreOptions.STORE); + this.checkOptionUnique(config, CoreOptions.STORE); /* - * NOTE: rocksdb can't use same data path for different graph, - * but it's not easy to check here + * TODO: should check data path for rocksdb since can't use the same + * data path for different graphs, but it's not easy to check here. */ - String backend = config.get(CoreOptions.BACKEND); - if (backend.equalsIgnoreCase("rocksdb")) { - // TODO: should check data path... - } } - private void checkOptionsUnique(HugeConfig config, - TypedOption option) { + private void checkOptionUnique(HugeConfig config, + TypedOption option) { Object incomingValue = config.get(option); for (String graphName : this.graphs.keySet()) { HugeGraph graph = this.graph(graphName); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java index d3a70a2748..69e495f56d 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java @@ -158,9 +158,10 @@ public interface HugeGraph extends Graph { public void createSnapshot(); public void resumeSnapshot(); + public void create(String configPath, Id server, NodeRole role); public void drop(); - public HugeConfig cloneConfig(); + public HugeConfig cloneConfig(String newGraph); @Override public HugeFeatures features(); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java index eaa760ac1f..6b4b205ede 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java @@ -906,10 +906,22 @@ public synchronized void close() throws Exception { this.name); } + @Override + public void create(String configPath, Id server, NodeRole role) { + this.initBackend(); + this.serverStarted(server, role); + + // Write config to disk file + ConfigUtil.writeToFile(configPath, this.name(), this.configuration()); + } + @Override public void drop() { this.clearBackend(); - ConfigUtil.deleteFile(this.configuration().getFile()); + + HugeConfig config = this.configuration(); + this.storeProvider.onDeleteConfig(config); + ConfigUtil.deleteFile(config.getFile()); try { /* @@ -926,8 +938,10 @@ public void drop() { } @Override - public HugeConfig cloneConfig() { + public HugeConfig cloneConfig(String newGraph) { HugeConfig config = (HugeConfig) this.configuration().clone(); + config.setDelimiterParsingDisabled(true); + this.storeProvider.onCloneConfig(config, newGraph); return config; } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java index fc35b7fb84..d5eaf5a407 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java @@ -28,6 +28,8 @@ import com.baidu.hugegraph.HugeGraph; import com.baidu.hugegraph.backend.BackendException; import com.baidu.hugegraph.backend.store.raft.StoreSnapshotFile; +import com.baidu.hugegraph.config.CoreOptions; +import com.baidu.hugegraph.config.HugeConfig; import com.baidu.hugegraph.event.EventHub; import com.baidu.hugegraph.event.EventListener; import com.baidu.hugegraph.util.E; @@ -208,4 +210,14 @@ public BackendStore loadSystemStore(String name) { public EventHub storeEventHub() { return this.storeEventHub; } + + @Override + public void onCloneConfig(HugeConfig config, String newGraph) { + config.setProperty(CoreOptions.STORE.name(), newGraph); + } + + @Override + public void onDeleteConfig(HugeConfig config) { + // pass + } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java index 12058d7b73..3a1d954c12 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java @@ -20,6 +20,7 @@ package com.baidu.hugegraph.backend.store; import com.baidu.hugegraph.HugeGraph; +import com.baidu.hugegraph.config.HugeConfig; import com.baidu.hugegraph.event.EventHub; import com.baidu.hugegraph.event.EventListener; @@ -63,4 +64,8 @@ public interface BackendStoreProvider { public void unlisten(EventListener listener); public EventHub storeEventHub(); + + public void onCloneConfig(HugeConfig config, String newGraph); + + public void onDeleteConfig(HugeConfig config); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java index b8d8bae370..6c20db44f8 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java @@ -31,6 +31,7 @@ import com.baidu.hugegraph.backend.store.BackendStoreSystemInfo; import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.StoreAction; import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.StoreType; +import com.baidu.hugegraph.config.HugeConfig; import com.baidu.hugegraph.event.EventHub; import com.baidu.hugegraph.event.EventListener; import com.baidu.hugegraph.util.E; @@ -220,6 +221,16 @@ public void createSnapshot() { LOG.debug("Graph '{}' has writed snapshot", this.graph()); } + @Override + public void onCloneConfig(HugeConfig config, String newGraph) { + this.provider.onCloneConfig(config, newGraph); + } + + @Override + public void onDeleteConfig(HugeConfig config) { + this.provider.onDeleteConfig(config); + } + @Override public void resumeSnapshot() { // Jraft doesn't expose API to load snapshot diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/ConfigUtil.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/ConfigUtil.java index aeea9b3d84..0b80561ac3 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/util/ConfigUtil.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/util/ConfigUtil.java @@ -70,7 +70,7 @@ public static void checkGremlinConfig(String conf) { } public static Map scanGraphsDir(String graphsDirPath) { - LOG.info("Scaning graphs configuration directory {}", graphsDirPath); + LOG.info("Scaning option 'graphs' directory '{}'", graphsDirPath); File graphsDir = new File(graphsDirPath); E.checkArgument(graphsDir.exists() && graphsDir.isDirectory(), "Please ensure the path '%s' of option 'graphs' " + @@ -106,16 +106,19 @@ public static void writeToFile(String dir, String graphName, } public static void deleteFile(File file) { + if (!file.exists()) { + return; + } try { FileUtils.forceDelete(file); } catch (IOException e) { - throw new HugeException("Failed to delete HugeConfig file {}", + throw new HugeException("Failed to delete HugeConfig file '{}'", e, file); } } public static PropertiesConfiguration buildConfig(String configText) { - E.checkArgument(configText != null && !configText.isEmpty(), + E.checkArgument(StringUtils.isNotEmpty(configText), "The config text can't be null or empty"); PropertiesConfiguration propConfig = new PropertiesConfiguration(); try { diff --git a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStoreProvider.java b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStoreProvider.java index 2278060463..ca8ae701d9 100644 --- a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStoreProvider.java +++ b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStoreProvider.java @@ -19,10 +19,14 @@ package com.baidu.hugegraph.backend.store.rocksdb; +import java.io.File; + import com.baidu.hugegraph.backend.store.AbstractBackendStoreProvider; import com.baidu.hugegraph.backend.store.BackendStore; import com.baidu.hugegraph.backend.store.rocksdb.RocksDBStore.RocksDBGraphStore; import com.baidu.hugegraph.backend.store.rocksdb.RocksDBStore.RocksDBSchemaStore; +import com.baidu.hugegraph.config.HugeConfig; +import com.baidu.hugegraph.util.ConfigUtil; public class RocksDBStoreProvider extends AbstractBackendStoreProvider { @@ -40,6 +44,29 @@ protected BackendStore newGraphStore(String store) { return new RocksDBGraphStore(this, this.database(), store); } + @Override + public void onCloneConfig(HugeConfig config, String newGraph) { + super.onCloneConfig(config, newGraph); + + // NOTE: rocksdb can't use same data path for different graph + String suffix = "_" + newGraph; + String dataPath = config.get(RocksDBOptions.DATA_PATH); + config.setProperty(RocksDBOptions.DATA_PATH.name(), dataPath + suffix); + + String walPath = config.get(RocksDBOptions.WAL_PATH); + config.setProperty(RocksDBOptions.WAL_PATH.name(), walPath + suffix); + } + + @Override + public void onDeleteConfig(HugeConfig config) { + super.onDeleteConfig(config); + + String dataPath = config.get(RocksDBOptions.DATA_PATH); + String walPath = config.get(RocksDBOptions.WAL_PATH); + ConfigUtil.deleteFile(new File(dataPath)); + ConfigUtil.deleteFile(new File(walPath)); + } + @Override public String type() { return "rocksdb";