Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Prepare 0.3.2 #777

Merged
merged 6 commits into from
Dec 21, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
run: |
find . -type f -name "log4j.*" -exec rm -fv '{}' \;
find . -type f -name "simplelogger.*" -exec rm -fv '{}' \;
mvn -q --batch-mode --global-toolchains .github/toolchains.xml -Drelease \
-Panalysis verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar
continue-on-error: true
4 changes: 2 additions & 2 deletions .github/workflows/benchmark.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ jobs:
find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ steps.version.outputs.value }}|g' \
-e 's|^\( <version>\).*\(</version>\)$|\1${{ steps.version.outputs.value }}\2|' \
-e 's|${parent.groupId}|com.clickhouse|g' '{}' \;
find . -type f -name "log4j.*" -exec rm -fv '{}' \;
find . -type f -name "simplelogger.*" -exec rm -fv '{}' \;
- name: Build project
run: |
mvn --batch-mode --update-snapshots -q -DskipTests install
Expand Down Expand Up @@ -100,7 +100,7 @@ jobs:
find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ github.event.inputs.driver }}|g' \
-e 's|^\( <version>\).*\(</version>\)$|\1${{ github.event.inputs.driver }}\2|' \
-e 's|${parent.groupId}|com.clickhouse|g' -e 's|${project.parent.groupId}|com.clickhouse|g' '{}' \;
find . -type f -name "log4j.*" -exec rm -fv '{}' \;
find . -type f -name "simplelogger.*" -exec rm -fv '{}' \;
continue-on-error: true
- name: Install driver as needed
run: mvn --batch-mode --update-snapshots -q -DskipTests install
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ jobs:
find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ github.event.inputs.version }}|g' \
-e 's|^\( <version>\).*\(</version>\)$|\1${{ github.event.inputs.version }}\2|' \
-e 's|${parent.groupId}|com.clickhouse|g' -e 's|${project.parent.groupId}|com.clickhouse|g' '{}' \;
find . -type f -name "log4j.*" -exec rm -fv '{}' \;
find . -type f -name "simplelogger.*" -exec rm -fv '{}' \;
- name: Release Maven package
uses: samuelmeuli/action-maven-publish@v1
with:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/third_party_libs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ github.event.inputs.version }}|g' \
-e 's|^\( <version>\).*\(</version>\)$|\1${{ github.event.inputs.version }}\2|' \
-e 's|${parent.groupId}|com.clickhouse|g' -e 's|${project.parent.groupId}|com.clickhouse|g' '{}' \;
find . -type f -name "log4j.*" -exec rm -fv '{}' \;
find . -type f -name "simplelogger.*" -exec rm -fv '{}' \;
- name: Release Maven package
uses: samuelmeuli/action-maven-publish@v1
with:
Expand Down
2 changes: 1 addition & 1 deletion clickhouse-benchmark/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

<properties>
<clickhouse4j-driver.version>1.4.4</clickhouse4j-driver.version>
<native-driver.version>2.6.1</native-driver.version>
<native-driver.version>2.6.3</native-driver.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<jmh.version>1.33</jmh.version>
<shade.name>benchmarks</shade.name>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,11 @@ public enum JdbcDriver {
"jdbc:clickhouse://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s",
Constants.HTTP_PORT),
// ClickHouse JDBC Driver
ClickhouseHttpJdbc("com.clickhouse.jdbc.ClickHouseDriver",
"jdbc:ch://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s",
ClickhouseHttpJdbc1("com.clickhouse.jdbc.ClickHouseDriver",
"jdbc:ch://%s:%s/%s?http_connection_provider=HTTP_URL_CONNECTION&ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s",
Constants.HTTP_PORT),
ClickhouseHttpJdbc2("com.clickhouse.jdbc.ClickHouseDriver",
"jdbc:ch://%s:%s/%s?http_connection_provider=HTTP_CLIENT&ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s",
Constants.HTTP_PORT),
ClickhouseGrpcJdbc("com.clickhouse.jdbc.ClickHouseDriver",
"jdbc:ch:grpc://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&max_inbound_message_size=2147483647&compress=%s",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
import com.clickhouse.benchmark.BaseState;
import com.clickhouse.client.ClickHouseEnum;
import com.clickhouse.client.ClickHouseValue;
import com.clickhouse.client.data.ClickHouseBigDecimalValue;
import com.clickhouse.client.data.ClickHouseBigIntegerValue;
Expand Down Expand Up @@ -82,7 +83,7 @@ public void setupSamples() {
add(map, list, Double.class, () -> ClickHouseDoubleValue.ofNull());
add(map, list, BigInteger.class, () -> ClickHouseBigIntegerValue.ofNull());
add(map, list, BigDecimal.class, () -> ClickHouseBigDecimalValue.ofNull());
add(map, list, Enum.class, () -> ClickHouseEnumValue.ofNull());
add(map, list, Enum.class, () -> ClickHouseEnumValue.ofNull(ClickHouseEnum.EMPTY));
add(map, list, Inet4Address.class, () -> ClickHouseIpv4Value.ofNull());
add(map, list, Inet6Address.class, () -> ClickHouseIpv6Value.ofNull());

Expand Down Expand Up @@ -121,7 +122,7 @@ ClickHouseValue newValue(Class<?> clazz) {
} else if (BigDecimal.class.equals(clazz)) {
return ClickHouseBigDecimalValue.ofNull();
} else if (Enum.class.equals(clazz)) {
return ClickHouseEnumValue.ofNull();
return ClickHouseEnumValue.ofNull(ClickHouseEnum.EMPTY);
} else if (Inet4Address.class.equals(clazz)) {
return ClickHouseIpv4Value.ofNull();
} else if (Inet6Address.class.equals(clazz)) {
Expand Down
2 changes: 1 addition & 1 deletion clickhouse-client/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@

<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<artifactId>slf4j-simple</artifactId>
<scope>test</scope>
</dependency>
<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -170,9 +170,8 @@ public void init(ClickHouseConfig config) {
this.config = config;
if (this.executor == null) { // only initialize once
int threads = config.getMaxThreadsPerClient();
this.executor = threads <= 0 ? ClickHouseClient.getExecutorService()
: ClickHouseUtils.newThreadPool(getClass().getSimpleName(), threads,
config.getMaxQueuedRequests());
this.executor = threads < 1 ? ClickHouseClient.getExecutorService()
: ClickHouseUtils.newThreadPool(this, threads, config.getMaxQueuedRequests());
}

initialized = true;
Expand All @@ -196,27 +195,16 @@ public final void close() {
try {
server = null;

if (executor != null) {
executor.shutdown();
}

if (connection != null) {
closeConnection(connection, false);
connection = null;
}

// shutdown* won't shutdown commonPool, so awaitTermination will always time out
// on the other hand, for a client-specific thread pool, we'd better shut it
// down for real
if (executor != null && config.getMaxThreadsPerClient() > 0
&& !executor.awaitTermination(config.getConnectionTimeout(), TimeUnit.MILLISECONDS)) {
executor.shutdownNow();
// avoid shutting down shared thread pool
if (executor != null && config.getMaxThreadsPerClient() > 0 && !executor.isTerminated()) {
executor.shutdown();
}

executor = null;
connection = null;
} catch (InterruptedException e) {
log.warn("Got interrupted when closing client", e);
Thread.currentThread().interrupt();
} catch (Exception e) {
log.warn("Exception occurred when closing client", e);
} finally {
Expand All @@ -226,7 +214,7 @@ public final void close() {
closeConnection(connection, true);
}

if (executor != null) {
if (executor != null && config.getMaxThreadsPerClient() > 0) {
executor.shutdownNow();
}
} finally {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import java.util.Objects;
import java.util.ServiceLoader;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ForkJoinPool;

import com.clickhouse.client.config.ClickHouseOption;
import com.clickhouse.client.config.ClickHouseDefaults;
Expand All @@ -23,23 +22,17 @@ public class ClickHouseClientBuilder {
static {
int maxThreads = (int) ClickHouseDefaults.MAX_THREADS.getEffectiveDefaultValue();
int maxRequests = (int) ClickHouseDefaults.MAX_REQUESTS.getEffectiveDefaultValue();
long keepAliveTimeoutMs = (long) ClickHouseDefaults.THREAD_KEEPALIVE_TIMEOUT.getEffectiveDefaultValue();

if (maxThreads <= 0 && maxRequests <= 0) {
// java -XX:+UnlockDiagnosticVMOptions -XX:NativeMemoryTracking=summary
// -XX:+PrintNMTStatistics -version
defaultExecutor = ForkJoinPool.commonPool();
} else {
if (maxThreads <= 0) {
maxThreads = Runtime.getRuntime().availableProcessors();
}

if (maxRequests <= 0) {
maxRequests = 0;
}

defaultExecutor = ClickHouseUtils.newThreadPool(ClickHouseClient.class.getSimpleName(), maxThreads,
maxRequests);
if (maxThreads <= 0) {
maxThreads = Runtime.getRuntime().availableProcessors();
}
if (maxRequests <= 0) {
maxRequests = 0;
}

defaultExecutor = ClickHouseUtils.newThreadPool(ClickHouseClient.class.getSimpleName(), maxThreads,
maxThreads * 2, maxRequests, keepAliveTimeoutMs, false);
}

protected ClickHouseConfig config;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,69 +36,76 @@ public final class ClickHouseColumn implements Serializable {
private int scale;
private List<ClickHouseColumn> nested;
private List<String> parameters;
private ClickHouseEnum enumConstants;

private int arrayLevel;
private ClickHouseColumn arrayBaseColumn;

private static ClickHouseColumn update(ClickHouseColumn column) {
column.enumConstants = ClickHouseEnum.EMPTY;
int size = column.parameters.size();
switch (column.dataType) {
case Array:
column.arrayLevel = 1;
column.arrayBaseColumn = column.nested.get(0);
while (column.arrayLevel < 255) {
if (column.arrayBaseColumn.dataType == ClickHouseDataType.Array) {
column.arrayLevel++;
column.arrayBaseColumn = column.arrayBaseColumn.nested.get(0);
} else {
break;
case Array:
column.arrayLevel = 1;
column.arrayBaseColumn = column.nested.get(0);
while (column.arrayLevel < 255) {
if (column.arrayBaseColumn.dataType == ClickHouseDataType.Array) {
column.arrayLevel++;
column.arrayBaseColumn = column.arrayBaseColumn.nested.get(0);
} else {
break;
}
}
}
break;
case DateTime:
if (size >= 2) { // same as DateTime64
column.scale = Integer.parseInt(column.parameters.get(0));
column.timeZone = TimeZone.getTimeZone(column.parameters.get(1).replace("'", ""));
} else if (size == 1) { // same as DateTime32
// unfortunately this will fall back to GMT if the time zone
// cannot be resolved
TimeZone tz = TimeZone.getTimeZone(column.parameters.get(0).replace("'", ""));
column.timeZone = tz;
}
break;
case DateTime32:
if (size > 0) {
// unfortunately this will fall back to GMT if the time zone
// cannot be resolved
TimeZone tz = TimeZone.getTimeZone(column.parameters.get(0).replace("'", ""));
column.timeZone = tz;
}
break;
case DateTime64:
if (size > 0) {
break;
case Enum:
case Enum8:
case Enum16:
column.enumConstants = new ClickHouseEnum(column.parameters);
break;
case DateTime:
if (size >= 2) { // same as DateTime64
column.scale = Integer.parseInt(column.parameters.get(0));
column.timeZone = TimeZone.getTimeZone(column.parameters.get(1).replace("'", ""));
} else if (size == 1) { // same as DateTime32
// unfortunately this will fall back to GMT if the time zone
// cannot be resolved
TimeZone tz = TimeZone.getTimeZone(column.parameters.get(0).replace("'", ""));
column.timeZone = tz;
}
break;
case DateTime32:
if (size > 0) {
// unfortunately this will fall back to GMT if the time zone
// cannot be resolved
TimeZone tz = TimeZone.getTimeZone(column.parameters.get(0).replace("'", ""));
column.timeZone = tz;
}
break;
case DateTime64:
if (size > 0) {
column.scale = Integer.parseInt(column.parameters.get(0));
}
if (size > 1) {
column.timeZone = TimeZone.getTimeZone(column.parameters.get(1).replace("'", ""));
}
break;
case Decimal:
if (size >= 2) {
column.precision = Integer.parseInt(column.parameters.get(0));
column.scale = Integer.parseInt(column.parameters.get(1));
}
break;
case Decimal32:
case Decimal64:
case Decimal128:
case Decimal256:
column.scale = Integer.parseInt(column.parameters.get(0));
}
if (size > 1) {
column.timeZone = TimeZone.getTimeZone(column.parameters.get(1).replace("'", ""));
}
break;
case Decimal:
if (size >= 2) {
break;
case FixedString:
column.precision = Integer.parseInt(column.parameters.get(0));
column.scale = Integer.parseInt(column.parameters.get(1));
}
break;
case Decimal32:
case Decimal64:
case Decimal128:
case Decimal256:
column.scale = Integer.parseInt(column.parameters.get(0));
break;
case FixedString:
column.precision = Integer.parseInt(column.parameters.get(0));
break;
default:
break;
break;
default:
break;
}

return column;
Expand Down Expand Up @@ -395,6 +402,11 @@ public boolean isArray() {
return dataType == ClickHouseDataType.Array;
}

public boolean isEnum() {
return dataType == ClickHouseDataType.Enum || dataType == ClickHouseDataType.Enum8
|| dataType == ClickHouseDataType.Enum16;
}

public boolean isMap() {
return dataType == ClickHouseDataType.Map;
}
Expand All @@ -419,6 +431,10 @@ public ClickHouseDataType getDataType() {
return dataType;
}

public ClickHouseEnum getEnumConstants() {
return enumConstants;
}

public String getOriginalTypeName() {
return originalTypeName;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,14 @@ public enum ClickHouseDataType {
UInt64(Long.class, false, true, false, 8, 20, 0, 0, 0, "BIGINT UNSIGNED"),
UInt128(BigInteger.class, false, true, false, 16, 39, 0, 0, 0),
UInt256(BigInteger.class, false, true, false, 32, 78, 0, 0, 0), Int8(Byte.class, false, true, true, 1, 3, 0, 0, 0,
"BOOL", "BOOLEAN", "BYTE", "INT1", "INT1 SIGNED", "TINYINT", "TINYINT SIGNED"),
"BYTE", "INT1", "INT1 SIGNED", "TINYINT", "TINYINT SIGNED"),
Int16(Short.class, false, true, true, 2, 5, 0, 0, 0, "SMALLINT", "SMALLINT SIGNED"),
Int32(Integer.class, false, true, true, 4, 10, 0, 0, 0, "INT", "INTEGER", "MEDIUMINT", "INT SIGNED",
"INTEGER SIGNED", "MEDIUMINT SIGNED"),
Int64(Long.class, false, true, true, 8, 19, 0, 0, 0, "BIGINT", "BIGINT SIGNED"),
Int128(BigInteger.class, false, true, true, 16, 39, 0, 0, 0),
Int256(BigInteger.class, false, true, true, 32, 77, 0, 0, 0),
Bool(Boolean.class, false, false, true, 1, 3, 0, 0, 0, "BOOLEAN"),
Date(LocalDate.class, false, false, false, 2, 10, 0, 0, 0),
Date32(LocalDate.class, false, false, false, 4, 10, 0, 0, 0),
DateTime(LocalDateTime.class, true, false, false, 0, 29, 0, 0, 9, "TIMESTAMP"),
Expand Down
Loading