Skip to content

Commit

Permalink
Upgrade Pinot Libraries to 1.1.0
Browse files Browse the repository at this point in the history
  • Loading branch information
@elonazoulay @wendigo
elonazoulay authored and wendigo committed Mar 25, 2024
1 parent 3b878b1 commit e961285
Show file tree
Hide file tree
Showing 9 changed files with 133 additions and 131 deletions.
130 changes: 57 additions & 73 deletions plugin/trino-pinot/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,32 @@

<properties>
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
<dep.pinot.version>0.12.1</dep.pinot.version>
<dep.pinot.version>1.1.0</dep.pinot.version>
<!--
Project's default for air.test.parallel is 'methods'. By design, 'instances' makes TestNG run tests from one class in a single thread.
As a side effect, it prevents TestNG from initializing multiple test instances upfront, which happens with 'methods'.
A potential downside can be long tail single-threaded execution of a single long test class.
TODO (https://github.com/trinodb/trino/issues/11294) remove when we upgrade to surefire with https://issues.apache.org/jira/browse/SUREFIRE-1967
-->
<air.test.parallel>instances</air.test.parallel>
<air.test.jvm.additional-arguments>${air.test.jvm.additional-arguments.default}
--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED
--add-exports=java.base/sun.nio.ch=ALL-UNNAMED
--add-exports=jdk.unsupported/sun.misc=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
--add-opens=jdk.compiler/com.sun.tools.javac=ALL-UNNAMED
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
--add-opens=java.base/java.io=ALL-UNNAMED
--add-opens=java.base/java.util=ALL-UNNAMED</air.test.jvm.additional-arguments>
</properties>

<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
<version>1.32.0</version>
<groupId>net.openhft</groupId>
<artifactId>posix</artifactId>
<version>2.25ea0</version>
</dependency>
</dependencies>
</dependencyManagement>
Expand Down Expand Up @@ -118,10 +135,15 @@
<artifactId>jakarta.validation-api</artifactId>
</dependency>

<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>

<dependency>
<groupId>org.apache.helix</groupId>
<artifactId>helix-core</artifactId>
<version>1.0.4</version>
<version>1.3.1</version>
<exclusions>
<exclusion>
<groupId>commons-io</groupId>
Expand All @@ -135,6 +157,10 @@
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
Expand All @@ -148,10 +174,6 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
</exclusion>
</exclusions>
</dependency>

Expand All @@ -164,14 +186,6 @@
<groupId>com.101tec</groupId>
<artifactId>zkclient</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.code.findbugs</groupId>
<artifactId>annotations</artifactId>
Expand All @@ -192,14 +206,6 @@
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>jakarta.annotation</groupId>
<artifactId>jakarta.annotation-api</artifactId>
</exclusion>
<exclusion>
<groupId>jakarta.ws.rs</groupId>
<artifactId>jakarta.ws.rs-api</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
Expand All @@ -224,10 +230,6 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
Expand All @@ -244,10 +246,18 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j2-impl</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
Expand Down Expand Up @@ -280,10 +290,6 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
</exclusion>
</exclusions>
</dependency>

Expand All @@ -292,26 +298,10 @@
<artifactId>pinot-core</artifactId>
<version>${dep.pinot.version}</version>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>jakarta.annotation</groupId>
<artifactId>jakarta.annotation-api</artifactId>
</exclusion>
<exclusion>
<groupId>jakarta.ws.rs</groupId>
<artifactId>jakarta.ws.rs-api</artifactId>
</exclusion>
<exclusion>
<groupId>javax.validation</groupId>
<artifactId>validation-api</artifactId>
Expand All @@ -329,16 +319,16 @@
<artifactId>kafka_2.10</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j2-impl</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-sandbox</artifactId>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
Expand All @@ -364,6 +354,10 @@
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-grizzly2-http</artifactId>
</exclusion>
<exclusion>
<groupId>org.locationtech.jts</groupId>
<artifactId>jts-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
Expand All @@ -379,22 +373,18 @@
<groupId>org.apache.pinot</groupId>
<artifactId>pinot-segment-local</artifactId>
<version>${dep.pinot.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.pinot</groupId>
<artifactId>pinot-segment-spi</artifactId>
<version>${dep.pinot.version}</version>
<exclusions>
<exclusion>
<groupId>org.locationtech.jts</groupId>
<artifactId>jts-core</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
Expand All @@ -415,8 +405,8 @@
<artifactId>log4j-slf4j-impl</artifactId>
</exclusion>
<exclusion>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j2-impl</artifactId>
</exclusion>
</exclusions>
</dependency>
Expand Down Expand Up @@ -480,12 +470,6 @@
<scope>runtime</scope>
</dependency>

<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<scope>runtime</scope>
</dependency>

<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
Expand Down
10 changes: 5 additions & 5 deletions plugin/trino-pinot/src/main/java/io/trino/plugin/pinot/client/PinotLegacyDataFetcher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import io.trino.plugin.pinot.PinotSessionProperties;
import io.trino.plugin.pinot.PinotSplit;
import io.trino.spi.connector.ConnectorSession;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.datatable.DataTable;
import org.apache.pinot.common.metrics.BrokerMetrics;
import org.apache.pinot.common.request.BrokerRequest;
Expand All @@ -41,7 +42,6 @@

import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
Expand Down Expand Up @@ -218,12 +218,12 @@ public Iterator<PinotDataTableWithSize> queryPinot(ConnectorSession session, Str
throw new PinotException(PINOT_INVALID_PQL_GENERATED, Optional.of(query), format("Parsing error with on %s, Error = %s", serverHost, e.getMessage()), e);
}
ServerInstance serverInstance = pinotHostMapper.getServerInstance(serverHost);
Map<ServerInstance, List<String>> routingTable = new HashMap<>();
routingTable.put(serverInstance, new ArrayList<>(segments));
Map<ServerInstance, Pair<List<String>, List<String>>> routingTable = new HashMap<>();
routingTable.put(serverInstance, Pair.of(segments, segments));
String tableName = brokerRequest.getQuerySource().getTableName();
String rawTableName = TableNameBuilder.extractRawTableName(tableName);
Map<ServerInstance, List<String>> offlineRoutingTable = TableNameBuilder.isOfflineTableResource(tableName) ? routingTable : null;
Map<ServerInstance, List<String>> realtimeRoutingTable = TableNameBuilder.isRealtimeTableResource(tableName) ? routingTable : null;
Map<ServerInstance, Pair<List<String>, List<String>>> offlineRoutingTable = TableNameBuilder.isOfflineTableResource(tableName) ? routingTable : null;
Map<ServerInstance, Pair<List<String>, List<String>>> realtimeRoutingTable = TableNameBuilder.isRealtimeTableResource(tableName) ? routingTable : null;
BrokerRequest offlineBrokerRequest = TableNameBuilder.isOfflineTableResource(tableName) ? brokerRequest : null;
BrokerRequest realtimeBrokerRequest = TableNameBuilder.isRealtimeTableResource(tableName) ? brokerRequest : null;
AsyncQueryResponse asyncQueryResponse =
Expand Down
18 changes: 9 additions & 9 deletions plugin/trino-pinot/src/main/java/io/trino/plugin/pinot/query/PinotExpressionRewriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,9 @@
import static java.lang.String.format;
import static java.util.Locale.ENGLISH;
import static java.util.Objects.requireNonNull;
import static org.apache.pinot.common.function.TransformFunctionType.DATETIMECONVERT;
import static org.apache.pinot.common.function.TransformFunctionType.DATETRUNC;
import static org.apache.pinot.common.function.TransformFunctionType.TIMECONVERT;
import static org.apache.pinot.common.function.TransformFunctionType.DATE_TIME_CONVERT;
import static org.apache.pinot.common.function.TransformFunctionType.DATE_TRUNC;
import static org.apache.pinot.common.function.TransformFunctionType.TIME_CONVERT;
import static org.apache.pinot.common.request.Literal.stringValue;
import static org.apache.pinot.common.request.context.ExpressionContext.Type.FUNCTION;
import static org.apache.pinot.common.request.context.ExpressionContext.Type.IDENTIFIER;
Expand All @@ -80,9 +80,9 @@ private PinotExpressionRewriter() {}

static {
Map<TransformFunctionType, RewriteRule<FunctionContext>> functionMap = new HashMap<>();
functionMap.put(DATETIMECONVERT, new DateTimeConvertRewriteRule());
functionMap.put(TIMECONVERT, new TimeConvertRewriteRule());
functionMap.put(DATETRUNC, new DateTruncRewriteRule());
functionMap.put(DATE_TIME_CONVERT, new DateTimeConvertRewriteRule());
functionMap.put(TIME_CONVERT, new TimeConvertRewriteRule());
functionMap.put(DATE_TRUNC, new DateTruncRewriteRule());
FUNCTION_RULE_MAP = immutableEnumMap(functionMap);

Map<AggregationFunctionType, RewriteRule<FunctionContext>> aggregationFunctionMap = new HashMap<>();
Expand Down Expand Up @@ -164,7 +164,7 @@ private static class DateTimeConvertRewriteRule
@Override
public Pattern<FunctionContext> getPattern()
{
return transformFunction().with(transformFunctionType().equalTo(DATETIMECONVERT));
return transformFunction().with(transformFunctionType().equalTo(DATE_TIME_CONVERT));
}

@Override
Expand Down Expand Up @@ -197,7 +197,7 @@ private static class TimeConvertRewriteRule
@Override
public Pattern<FunctionContext> getPattern()
{
return transformFunction().with(transformFunctionType().equalTo(TIMECONVERT));
return transformFunction().with(transformFunctionType().equalTo(TIME_CONVERT));
}

@Override
Expand Down Expand Up @@ -228,7 +228,7 @@ private static class DateTruncRewriteRule
@Override
public Pattern<FunctionContext> getPattern()
{
return transformFunction().with(transformFunctionType().equalTo(DATETRUNC));
return transformFunction().with(transformFunctionType().equalTo(DATE_TRUNC));
}

@Override
Expand Down
2 changes: 1 addition & 1 deletion ...o-pinot/src/main/java/io/trino/plugin/pinot/query/PinotTransformFunctionTypeResolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ private PinotTransformFunctionTypeResolver() {}
{
Map<String, TransformFunctionType> builder = new HashMap<>();
for (TransformFunctionType transformFunctionType : TransformFunctionType.values()) {
for (String alias : transformFunctionType.getAliases()) {
for (String alias : transformFunctionType.getAlternativeNames()) {
TransformFunctionType previousValue = builder.put(canonicalize(alias), transformFunctionType);
checkState(previousValue == null || previousValue == transformFunctionType, "Duplicate key with different values for alias '%s', transform function type '%s' and previous value '%s'", canonicalize(alias), transformFunctionType, previousValue);
}
Expand Down
Loading

0 comments on commit e961285

Please sign in to comment.