Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refact: clean extra store file in all modules #434

Merged
merged 8 commits into from
Mar 25, 2023
Merged

refact: clean extra store file in all modules #434

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
7bbfc03
chore: clean store file in all modules
imbajin Mar 1, 2023
524faba
remove useless license files & download parquet test file
imbajin Mar 1, 2023
5cfd8f8
Delete yarn.lock
imbajin Mar 1, 2023
ce9f3ce
Merge branch 'master' into fix-binary
imbajin Mar 16, 2023
ff5c6d0
use a common prefix for the URL
imbajin Mar 16, 2023
38993bd
Merge branch 'master' into fix-binary
imbajin Mar 24, 2023
010367c
Update FileLoadTest.java
imbajin Mar 24, 2023
87c95a2
Update FileLoadTest.java
imbajin Mar 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
[![Build Status](https://github.com/apache/hugegraph-toolchain/actions/workflows/hubble-ci.yml/badge.svg)](https://github.com/apache/hugegraph-toolchain/actions/workflows/hubble-ci.yml)
[![Build Status](https://github.com/apache/hugegraph-toolchain/actions/workflows/tools-ci.yml/badge.svg)](https://github.com/apache/hugegraph-toolchain/actions/workflows/tools-ci.yml)
[![codecov](https://codecov.io/gh/apache/hugegraph-toolchain/branch/master/graph/badge.svg)](https://codecov.io/gh/apache/hugegraph-toolchain)
[![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.apache.hugegraph/hugegraph-loader/badge.svg)](https://mvnrepository.com/artifact/org.apache.hugegraph/hugegraph-loader)
[![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.apache.hugegraph/hugegraph-client/badge.svg)](https://mvnrepository.com/artifact/org.apache.hugegraph/hugegraph-client)

`hugegraph-toolchain` is the integration project of a series of utilities for [HugeGraph](https://github.com/apache/hugegraph), it includes 4 main modules.

Expand Down
18 changes: 18 additions & 0 deletions hugegraph-client/src/main/java/org/apache/hugegraph/util/CommonUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,18 @@

package org.apache.hugegraph.util;

import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.Map;

import org.apache.commons.io.FileUtils;

public final class CommonUtil {

public static final String PREFIX = "https://github.com/apache/incubator-hugegraph-doc/" +
"raw/binary/dist/toolchain/";

public static void checkMapClass(Object object, Class<?> kClass,
Class<?> vClass) {
E.checkArgumentNotNull(object, "The object can't be null");
Expand All @@ -41,4 +49,14 @@ public static void checkMapClass(Object object, Class<?> kClass,
"but got '%s'(%s)", vClass, value, value.getClass());
}
}

public static void downloadFileByUrl(String url, String destPath) {
int connectTimeout = 5000;
int readTimeout = 5000;
try {
FileUtils.copyURLToFile(new URL(url), new File(destPath), connectTimeout, readTimeout);
} catch (IOException e) {
throw new RuntimeException("Failed to download file, please check the network", e);
}
}
}
21 changes: 14 additions & 7 deletions hugegraph-client/src/test/java/org/apache/hugegraph/functional/HugeClientHttpsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@
import org.apache.hugegraph.structure.constant.T;
import org.apache.hugegraph.structure.graph.Vertex;
import org.apache.hugegraph.testutil.Assert;
import org.apache.hugegraph.util.CommonUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import com.google.common.collect.ImmutableMap;
Expand All @@ -40,11 +42,16 @@ public class HugeClientHttpsTest {
private static final int MAX_CONNS_PER_ROUTE = 10;
private static final int MAX_CONNS = 10;
private static final int IDLE_TIME = 30;
private static final String TRUST_STORE_FILE = "src/test/resources/hugegraph.truststore";
private static final String TRUST_STORE_PATH = "src/test/resources/hugegraph.truststore";
private static final String TRUST_STORE_PASSWORD = "hugegraph";

private static HugeClient client;

@Before
public void init() {
CommonUtil.downloadFileByUrl(CommonUtil.PREFIX + "hugegraph.truststore", TRUST_STORE_PATH);
}

@After
public void teardown() throws Exception {
Assert.assertNotNull("Client is not opened", client);
Expand All @@ -55,7 +62,7 @@ public void teardown() throws Exception {
public void testHttpsClientBuilderWithConnection() {
client = HugeClient.builder(BASE_URL, GRAPH)
.configUser(USERNAME, PASSWORD)
.configSSL(TRUST_STORE_FILE, TRUST_STORE_PASSWORD)
.configSSL(TRUST_STORE_PATH, TRUST_STORE_PASSWORD)
.build();
Assert.assertTrue(client.graphs().listGraph().contains("hugegraph"));
this.addVertexAndCheckPropertyValue();
Expand All @@ -66,7 +73,7 @@ public void testHttpsClientWithConnectionPoolNoUserParam() {
client = HugeClient.builder(BASE_URL, GRAPH)
.configTimeout(TIMEOUT)
.configPool(MAX_CONNS, MAX_CONNS_PER_ROUTE)
.configSSL(TRUST_STORE_FILE, TRUST_STORE_PASSWORD)
.configSSL(TRUST_STORE_PATH, TRUST_STORE_PASSWORD)
.build();
Assert.assertTrue(client.graphs().listGraph().contains("hugegraph"));
this.addVertexAndCheckPropertyValue();
Expand All @@ -77,7 +84,7 @@ public void testHttpsClientWithConnectionPoolNoTimeOutParam() {
client = HugeClient.builder(BASE_URL, GRAPH)
.configUser(USERNAME, PASSWORD)
.configPool(MAX_CONNS, MAX_CONNS_PER_ROUTE)
.configSSL(TRUST_STORE_FILE, TRUST_STORE_PASSWORD)
.configSSL(TRUST_STORE_PATH, TRUST_STORE_PASSWORD)
.build();
Assert.assertTrue(client.graphs().listGraph().contains("hugegraph"));
this.addVertexAndCheckPropertyValue();
Expand All @@ -88,7 +95,7 @@ public void testHttpsClientNewBuilderWithConnectionNoPoolParam() {
client = HugeClient.builder(BASE_URL, GRAPH)
.configUser(USERNAME, PASSWORD)
.configTimeout(TIMEOUT)
.configSSL(TRUST_STORE_FILE, TRUST_STORE_PASSWORD)
.configSSL(TRUST_STORE_PATH, TRUST_STORE_PASSWORD)
.build();
Assert.assertTrue(client.graphs().listGraph().contains("hugegraph"));
this.addVertexAndCheckPropertyValue();
Expand All @@ -100,7 +107,7 @@ public void testHttpsClientNewBuilderWithConnectionPool() {
.configUser(USERNAME, PASSWORD)
.configTimeout(TIMEOUT)
.configPool(MAX_CONNS, MAX_CONNS_PER_ROUTE)
.configSSL(TRUST_STORE_FILE, TRUST_STORE_PASSWORD)
.configSSL(TRUST_STORE_PATH, TRUST_STORE_PASSWORD)
.configIdleTime(IDLE_TIME)
.build();
Assert.assertTrue(client.graphs().listGraph().contains("hugegraph"));
Expand All @@ -113,7 +120,7 @@ public void testHttpsClientNewBuilderZeroPoolParam() {
.configUser(USERNAME, PASSWORD)
.configTimeout(TIMEOUT)
.configPool(0, 0)
.configSSL(TRUST_STORE_FILE, TRUST_STORE_PASSWORD)
.configSSL(TRUST_STORE_PATH, TRUST_STORE_PASSWORD)
.build();
Assert.assertTrue(client.graphs().listGraph().contains("hugegraph"));
this.addVertexAndCheckPropertyValue();
Expand Down
5 changes: 1 addition & 4 deletions hugegraph-client/src/test/java/org/apache/hugegraph/testutil/Utils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,10 +135,7 @@ public static boolean equalPropertyKey(PropertyKey left,
if (left.dataType() != right.dataType()) {
return false;
}
if (left.cardinality() != right.cardinality()) {
return false;
}
return true;
return left.cardinality() == right.cardinality();
}

public static boolean equalVertexLabel(VertexLabel left,
Expand Down
Binary file removed hugegraph-client/src/test/resources/hugegraph.truststore
View file
Open in desktop
Binary file not shown.
5 changes: 2 additions & 3 deletions hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/SettingSSLService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,10 @@

package org.apache.hugegraph.service;

import org.springframework.stereotype.Service;

import org.apache.hugegraph.config.HugeConfig;
import org.apache.hugegraph.entity.GraphConnection;
import org.apache.hugegraph.options.HubbleOptions;
import org.springframework.stereotype.Service;

import lombok.extern.log4j.Log4j2;

Expand All @@ -32,7 +31,7 @@ public class SettingSSLService {

public void configSSL(HugeConfig config, GraphConnection connection) {
String protocol = config.get(HubbleOptions.SERVER_PROTOCOL);
if (protocol != null && protocol.equals("https")) {
if ("https".equals(protocol)) {
connection.setProtocol(protocol);
String trustStoreFile = config.get(
HubbleOptions.CLIENT_TRUSTSTORE_FILE);
Expand Down
Binary file removed hugegraph-hubble/hubble-be/src/main/resources/public-certs.store
View file
Open in desktop
Binary file not shown.
7 changes: 0 additions & 7 deletions hugegraph-hubble/hubble-be/src/main/resources/verify-license.json
View file
Open in desktop

This file was deleted.

Empty file removed hugegraph-hubble/yarn.lock
View file
Open in desktop
Empty file.
Binary file removed hugegraph-loader/assembly/static/conf/hugegraph.truststore
View file
Open in desktop
Binary file not shown.
Binary file removed hugegraph-loader/assembly/travis/conf/hugegraph.truststore
View file
Open in desktop
Binary file not shown.
2 changes: 1 addition & 1 deletion hugegraph-loader/src/main/java/org/apache/hugegraph/loader/constant/Constants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public final class Constants {
public static final String HTTPS_PREFIX = "https://";
public static final String JSON_SUFFIX = ".json";
public static final String GROOVY_SCHEMA = "schema";
public static final String TRUST_STORE_FILE = "conf/hugegraph.truststore";
public static final String TRUST_STORE_PATH = "conf/hugegraph.truststore";

public static final String FIELD_VERSION = "version";
public static final String V1_STRUCT_VERSION = "1.0";
Expand Down
4 changes: 1 addition & 3 deletions hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/HugeClientHolder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import java.nio.file.Paths;

import org.apache.commons.lang3.StringUtils;

import org.apache.hugegraph.driver.HugeClient;
import org.apache.hugegraph.driver.HugeClientBuilder;
import org.apache.hugegraph.exception.ServerException;
Expand Down Expand Up @@ -61,8 +60,7 @@ public static HugeClient create(LoadOptions options) {
"The system property 'loader.home.path' " +
"can't be null or empty when enable " +
"https protocol");
trustFile = Paths.get(homePath, Constants.TRUST_STORE_FILE)
.toString();
trustFile = Paths.get(homePath, Constants.TRUST_STORE_PATH).toString();
} else {
trustFile = options.trustStoreFile;
}
Expand Down
24 changes: 15 additions & 9 deletions hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/functional/FileLoadTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
import org.apache.hugegraph.structure.schema.PropertyKey;
import org.apache.hugegraph.testutil.Assert;
import org.apache.hugegraph.testutil.Whitebox;
import org.apache.hugegraph.util.CommonUtil;
import org.apache.hugegraph.util.LongEncoding;
import org.junit.After;
import org.junit.Before;
Expand Down Expand Up @@ -2424,11 +2425,13 @@ public void testParquetCompressFile() {
"--batch-insert-threads", "2",
"--test-mode", "true"
};
String path = configPath("parquet_compress_file/vertex_person.parquet");
CommonUtil.downloadFileByUrl(CommonUtil.PREFIX + "vertex_person.parquet", path);
if (this.ioUtil instanceof HDFSUtil) {
HDFSUtil hdfsUtil = (HDFSUtil) this.ioUtil;
hdfsUtil.copy(configPath(
"parquet_compress_file/vertex_person.parquet"),
"hdfs://localhost:8020/files/vertex_person.parquet");
CommonUtil.downloadFileByUrl(CommonUtil.PREFIX + "vertex_person.parquet",
"src/test/resources/parquet_compress_file/vertex_person.parquet");
hdfsUtil.copy(path, "hdfs://localhost:8020/files/vertex_person.parquet");
}
HugeGraphLoader.main(args);

Expand All @@ -2437,8 +2440,7 @@ public void testParquetCompressFile() {
}

@Test
public void testNumberAndDatePrimaryKeysEncoded()
throws java.text.ParseException {
public void testNumberAndDatePrimaryKeysEncoded() {
ioUtil.write("vertex_person.csv",
"id,name,age,city",
"100,marko,29,Beijing");
Expand Down Expand Up @@ -3043,14 +3045,16 @@ public void testHttpsClientValueMapping() {
ioUtil.write("vertex_person.csv",
"tiny,1,1,1",
"mary,2,2,2");

CommonUtil.downloadFileByUrl(FILE_URL, TRUST_STORE_PATH);
String[] args = new String[]{
"-f", structPath("value_mapping/struct.json"),
"-s", configPath("value_mapping/schema.groovy"),
"-g", GRAPH,
"-h", SERVER,
"-p", String.valueOf(HTTPS_PORT),
"--protocol", HTTPS_PROTOCOL,
"--trust-store-file", TRUST_STORE_FILE,
"--trust-store-file", TRUST_STORE_PATH,
"--trust-store-password", "hugegraph",
"--batch-insert-threads", "2",
"--test-mode", "true"
Expand All @@ -3060,7 +3064,7 @@ public void testHttpsClientValueMapping() {
HugeClient httpsClient = null;
try {
httpsClient = HugeClient.builder(HTTPS_URL, GRAPH)
.configSSL(TRUST_STORE_FILE, "hugegraph")
.configSSL(TRUST_STORE_PATH, "hugegraph")
.build();
List<Vertex> vertices = httpsClient.graph().listVertices();
Assert.assertEquals(2, vertices.size());
Expand All @@ -3074,14 +3078,16 @@ public void testHttpsHolderClientValueMapping() {
ioUtil.write("vertex_person.csv",
"marko,1,1,1",
"vadas,2,2,2");

CommonUtil.downloadFileByUrl(FILE_URL, TRUST_STORE_PATH);
String[] args = new String[]{
"-f", structPath("value_mapping/struct.json"),
"-s", configPath("value_mapping/schema.groovy"),
"-g", GRAPH,
"-h", SERVER,
"-p", String.valueOf(HTTPS_PORT),
"--protocol", HTTPS_PROTOCOL,
"--trust-store-file", TRUST_STORE_FILE,
"--trust-store-file", TRUST_STORE_PATH,
"--trust-store-password", "hugegraph",
"--batch-insert-threads", "2",
"--test-mode", "true"
Expand All @@ -3093,7 +3099,7 @@ public void testHttpsHolderClientValueMapping() {
options.port = HTTPS_PORT;
options.graph = GRAPH;
options.protocol = HTTPS_PROTOCOL;
options.trustStoreFile = TRUST_STORE_FILE;
options.trustStoreFile = TRUST_STORE_PATH;
options.trustStoreToken = "hugegraph";

HugeClient httpsClient = null;
Expand Down
5 changes: 3 additions & 2 deletions hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/functional/LoadTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@
import org.apache.hugegraph.structure.constant.T;
import org.apache.hugegraph.structure.graph.Edge;
import org.apache.hugegraph.structure.graph.Vertex;

import org.apache.hugegraph.testutil.Assert;
import org.apache.hugegraph.util.CommonUtil;

public class LoadTest {

Expand All @@ -43,7 +43,8 @@ public class LoadTest {
protected static final String URL = String.format("http://%s:%s", SERVER, PORT);
protected static final String HTTPS_URL = String.format("https://%s:%s", SERVER, HTTPS_PORT);
protected static final String HTTPS_PROTOCOL = "https";
protected static final String TRUST_STORE_FILE = "assembly/travis/conf/hugegraph.truststore";
protected static final String TRUST_STORE_PATH = "assembly/travis/conf/hugegraph.truststore";
protected static final String FILE_URL = CommonUtil.PREFIX + "hugegraph.truststore";
protected static final HugeClient CLIENT = HugeClient.builder(URL, GRAPH).build();

public static String configPath(String fileName) {
Expand Down
Binary file removed hugegraph-loader/src/test/resources/parquet_compress_file/vertex_person.parquet
View file
Open in desktop
Binary file not shown.
Binary file removed hugegraph-tools/assembly/bin/keystore
View file
Open in desktop
Binary file not shown.
Binary file removed hugegraph-tools/assembly/conf/hugegraph.truststore
View file
Open in desktop
Binary file not shown.
17 changes: 8 additions & 9 deletions hugegraph-tools/src/main/java/org/apache/hugegraph/base/ToolClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,11 @@

public class ToolClient {

private static final String DEFAULT_TRUST_STORE_FILE =
"conf/hugegraph.truststore";
private static final String DEFAULT_TRUST_STORE_FILE = "conf/hugegraph.truststore";
private static final String DEFAULT_TRUST_STORE_PASSWORD = "hugegraph";

private HugeClient client;
private ObjectMapper mapper;
private final HugeClient client;
private final ObjectMapper mapper;

public ToolClient(ConnectionInfo info) {
if (info.username == null) {
Expand Down Expand Up @@ -125,13 +124,13 @@ public void close() {

public static class ConnectionInfo {

private String url;
private String graph;
private final String url;
private final String graph;
private String username;
private String password;
private Integer timeout;
private String trustStoreFile;
private String trustStorePassword;
private final Integer timeout;
private final String trustStoreFile;
private final String trustStorePassword;

public ConnectionInfo(String url, String graph,
String username, String password,
Expand Down