From 95a957a228d2358c818ce6f0782bcff0c966ae71 Mon Sep 17 00:00:00 2001 From: Xingyuan Lin Date: Wed, 6 May 2020 20:31:40 -0700 Subject: [PATCH] Upgrade to PrestoSQL 333 Some major changes: - `SqlFunction`, `SqlScalarFunction` and `ScalarFunctionImplementation` have evolved in https://github.com/prestosql/presto/pull/1764 - `Metadata::getScalarFunctionImplementation` evolved in https://github.com/prestosql/presto/pull/1039 - Type signature parser was moved to presto-main in https://github.com/prestosql/presto/pull/1738 --- defaultEnvironment.gradle | 4 +- docs/transport-udfs-presto.patch | 74 +++++-------------- transportable-udfs-examples/build.gradle | 4 +- transportable-udfs-plugin/build.gradle | 2 +- .../transport/presto/PrestoFactory.java | 18 +++-- .../transport/presto/StdUdfWrapper.java | 53 +++++++------ .../transport/presto/data/PrestoMap.java | 2 +- 7 files changed, 63 insertions(+), 94 deletions(-) diff --git a/defaultEnvironment.gradle b/defaultEnvironment.gradle index ced4d3d5..c6b83602 100644 --- a/defaultEnvironment.gradle +++ b/defaultEnvironment.gradle @@ -10,8 +10,8 @@ subprojects { url "https://conjars.org/repo" } } - project.ext.setProperty('presto-version', '319') - project.ext.setProperty('airlift-slice-version', '0.33') + project.ext.setProperty('presto-version', '333') + project.ext.setProperty('airlift-slice-version', '0.38') project.ext.setProperty('spark-group', 'org.apache.spark') project.ext.setProperty('spark-version', '2.3.0') } diff --git a/docs/transport-udfs-presto.patch b/docs/transport-udfs-presto.patch index 3fa1a9b8..c29b1dd6 100644 --- a/docs/transport-udfs-presto.patch +++ b/docs/transport-udfs-presto.patch @@ -1,28 +1,5 @@ -diff --git a/presto-main/pom.xml b/presto-main/pom.xml -index 83a20a23fe..02afa5310f 100644 ---- a/presto-main/pom.xml -+++ b/presto-main/pom.xml -@@ -424,6 +424,18 @@ - test - - -+ -+ org.apache.maven.plugins -+ maven-jar-plugin -+ 2.2 -+ -+ -+ -+ test-jar -+ -+ -+ -+ - - - diff --git a/presto-main/src/main/java/io/prestosql/server/PluginManager.java b/presto-main/src/main/java/io/prestosql/server/PluginManager.java -index f02ceeab03..88de943bed 100644 +index abcd001031..053c17aeed 100644 --- a/presto-main/src/main/java/io/prestosql/server/PluginManager.java +++ b/presto-main/src/main/java/io/prestosql/server/PluginManager.java @@ -23,6 +23,7 @@ import io.prestosql.connector.ConnectorManager; @@ -31,17 +8,17 @@ index f02ceeab03..88de943bed 100644 import io.prestosql.metadata.MetadataManager; +import io.prestosql.metadata.SqlScalarFunction; import io.prestosql.security.AccessControlManager; + import io.prestosql.security.GroupProviderManager; import io.prestosql.server.security.PasswordAuthenticatorManager; - import io.prestosql.spi.Plugin; -@@ -52,6 +53,7 @@ import java.util.List; - import java.util.ServiceLoader; +@@ -54,6 +55,7 @@ import java.util.ServiceLoader; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; + import java.util.function.Supplier; +import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkState; import static io.prestosql.metadata.FunctionExtractor.extractFunctions; -@@ -62,8 +64,22 @@ import static java.util.Objects.requireNonNull; +@@ -64,8 +66,22 @@ import static java.util.Objects.requireNonNull; @ThreadSafe public class PluginManager { @@ -54,8 +31,8 @@ index f02ceeab03..88de943bed 100644 private static final ImmutableList SPI_PACKAGES = ImmutableList.builder() + // io.prestosql.metadata is required for SqlScalarFunction and FunctionRegistry classes + .add("io.prestosql.metadata.") -+ // io.prestosql.operator.scalar is required for ScalarFunctionImplementation -+ .add("io.prestosql.operator.scalar.") ++ // io.prestosql.operator. is required for ScalarFunctionImplementation and TypeSignatureParser ++ .add("io.prestosql.operator.") .add("io.prestosql.spi.") + // io.prestosql.type is required for TypeManager, and all supported types + .add("io.prestosql.type.") @@ -63,8 +40,8 @@ index f02ceeab03..88de943bed 100644 + .add("io.prestosql.util.") .add("com.fasterxml.jackson.annotation.") .add("io.airlift.slice.") - .add("io.airlift.units.") -@@ -155,7 +171,21 @@ public class PluginManager + .add("org.openjdk.jol.") +@@ -159,11 +175,22 @@ public class PluginManager { ServiceLoader serviceLoader = ServiceLoader.load(Plugin.class, pluginClassLoader); List plugins = ImmutableList.copyOf(serviceLoader); @@ -74,32 +51,21 @@ index f02ceeab03..88de943bed 100644 + pluginClassLoader); + List sqlScalarFunctions = ImmutableList.copyOf(sqlScalarFunctionsServiceLoader); + -+ checkState(!plugins.isEmpty() || !sqlScalarFunctions.isEmpty(), "No service providers of type %s or %s", -+ Plugin.class.getName(), SqlScalarFunction.class.getName()); ++ checkState(!plugins.isEmpty() || !sqlScalarFunctions.isEmpty(), "No service providers of type %s or %s", Plugin.class.getName(), SqlScalarFunction.class.getName()); + -+ installPlugins(plugins); -+ -+ registerSqlScalarFunctions(sqlScalarFunctions); -+ } -+ -+ private void installPlugins(List plugins) -+ { for (Plugin plugin : plugins) { log.info("Installing %s", plugin.getClass().getName()); - installPlugin(plugin); -@@ -215,6 +245,15 @@ public class PluginManager + installPlugin(plugin, pluginClassLoader::duplicate); } - } - -+ public void registerSqlScalarFunctions(List sqlScalarFunctions) -+ { ++ + for (SqlScalarFunction sqlScalarFunction : sqlScalarFunctions) { -+ log.info("Registering function %s(%s)", sqlScalarFunction.getSignature().getName(), sqlScalarFunction.getSignature().getArgumentTypes().stream().map(e -> e.toString()).collect( -+ Collectors.joining(", "))); ++ log.info("Registering function %s(%s)", ++ sqlScalarFunction.getFunctionMetadata().getSignature().getName(), ++ sqlScalarFunction.getFunctionMetadata().getSignature().getArgumentTypes().stream() ++ .map(e -> e.toString()) ++ .collect(Collectors.joining(", "))); + metadataManager.addFunctions(ImmutableList.of(sqlScalarFunction)); + } -+ } -+ - private URLClassLoader buildClassLoader(String plugin) - throws Exception - { + } + + public void installPlugin(Plugin plugin, Supplier duplicatePluginClassLoaderFactory) diff --git a/transportable-udfs-examples/build.gradle b/transportable-udfs-examples/build.gradle index 240ba4e5..f55fbd29 100644 --- a/transportable-udfs-examples/build.gradle +++ b/transportable-udfs-examples/build.gradle @@ -33,8 +33,8 @@ subprojects { url "https://conjars.org/repo" } } - project.ext.setProperty('presto-version', '319') - project.ext.setProperty('airlift-slice-version', '0.33') + project.ext.setProperty('presto-version', '333') + project.ext.setProperty('airlift-slice-version', '0.38') project.ext.setProperty('spark-group', 'org.apache.spark') project.ext.setProperty('spark-version', '2.3.0') } diff --git a/transportable-udfs-plugin/build.gradle b/transportable-udfs-plugin/build.gradle index f465e780..6f4ade36 100644 --- a/transportable-udfs-plugin/build.gradle +++ b/transportable-udfs-plugin/build.gradle @@ -27,7 +27,7 @@ def writeVersionInfo = { file -> ant.propertyfile(file: file) { entry(key: "transport-version", value: version) entry(key: "hive-version", value: '1.2.2') - entry(key: "presto-version", value: '319') + entry(key: "presto-version", value: '333') entry(key: "spark-version", value: '2.3.0') entry(key: "scala-version", value: '2.11.8') } diff --git a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/PrestoFactory.java b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/PrestoFactory.java index c5e0c53e..944523c3 100644 --- a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/PrestoFactory.java +++ b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/PrestoFactory.java @@ -6,6 +6,7 @@ package com.linkedin.transport.presto; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import com.linkedin.transport.api.StdFactory; import com.linkedin.transport.api.data.StdArray; import com.linkedin.transport.api.data.StdBoolean; @@ -25,18 +26,19 @@ import io.airlift.slice.Slices; import io.prestosql.metadata.BoundVariables; import io.prestosql.metadata.Metadata; -import io.prestosql.metadata.Signature; +import io.prestosql.metadata.OperatorNotFoundException; +import io.prestosql.metadata.ResolvedFunction; import io.prestosql.operator.scalar.ScalarFunctionImplementation; +import io.prestosql.spi.function.OperatorType; import io.prestosql.spi.type.ArrayType; import io.prestosql.spi.type.MapType; import io.prestosql.spi.type.RowType; import io.prestosql.spi.type.Type; -import io.prestosql.spi.type.TypeSignature; import java.util.List; import java.util.stream.Collectors; import static io.prestosql.metadata.SignatureBinder.*; - +import static io.prestosql.operator.TypeSignatureParser.parseTypeSignature; public class PrestoFactory implements StdFactory { @@ -104,10 +106,14 @@ public StdStruct createStruct(StdType stdType) { @Override public StdType createStdType(String typeSignature) { return PrestoWrapper.createStdType( - metadata.getType(applyBoundVariables(TypeSignature.parseTypeSignature(typeSignature), boundVariables))); + metadata.getType(applyBoundVariables(parseTypeSignature(typeSignature, ImmutableSet.of()), boundVariables))); + } + + public ScalarFunctionImplementation getScalarFunctionImplementation(ResolvedFunction resolvedFunction) { + return metadata.getScalarFunctionImplementation(resolvedFunction); } - public ScalarFunctionImplementation getScalarFunctionImplementation(Signature signature) { - return metadata.getScalarFunctionImplementation(signature); + public ResolvedFunction resolveOperator(OperatorType operatorType, List argumentTypes) throws OperatorNotFoundException { + return metadata.resolveOperator(operatorType, argumentTypes); } } diff --git a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/StdUdfWrapper.java b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/StdUdfWrapper.java index 20abf719..00d8468b 100644 --- a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/StdUdfWrapper.java +++ b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/StdUdfWrapper.java @@ -7,6 +7,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.primitives.Booleans; import com.linkedin.transport.api.StdFactory; import com.linkedin.transport.api.data.PlatformData; import com.linkedin.transport.api.data.StdData; @@ -23,7 +25,9 @@ import com.linkedin.transport.api.udf.TopLevelStdUDF; import com.linkedin.transport.typesystem.GenericTypeSignatureElement; import io.prestosql.metadata.BoundVariables; +import io.prestosql.metadata.FunctionArgumentDefinition; import io.prestosql.metadata.FunctionKind; +import io.prestosql.metadata.FunctionMetadata; import io.prestosql.metadata.Metadata; import io.prestosql.metadata.Signature; import io.prestosql.metadata.SqlScalarFunction; @@ -32,7 +36,6 @@ import io.prestosql.spi.classloader.ThreadContextClassLoader; import io.prestosql.spi.type.IntegerType; import io.prestosql.spi.type.Type; -import io.prestosql.spi.type.TypeSignature; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodType; @@ -48,7 +51,7 @@ import static io.prestosql.metadata.Signature.*; import static io.prestosql.metadata.SignatureBinder.*; -import static io.prestosql.spi.type.TypeSignature.*; +import static io.prestosql.operator.TypeSignatureParser.parseTypeSignature; import static io.prestosql.util.Reflection.*; // Suppressing argument naming convention for the evalInternal methods @@ -57,16 +60,26 @@ public abstract class StdUdfWrapper extends SqlScalarFunction { private static final int DEFAULT_REFRESH_INTERVAL_DAYS = 1; private static final int JITTER_FACTOR = 50; // to calculate jitter from delay - private String _functionDescription; protected StdUdfWrapper(StdUDF stdUDF) { - super(new Signature(((TopLevelStdUDF) stdUDF).getFunctionName(), FunctionKind.SCALAR, - getTypeVariableConstraintsForStdUdf(stdUDF), ImmutableList.of(), - parseTypeSignature(stdUDF.getOutputParameterSignature()), stdUDF.getInputParameterSignatures() - .stream() - .map(TypeSignature::parseTypeSignature) - .collect(Collectors.toList()), false)); - _functionDescription = ((TopLevelStdUDF) stdUDF).getFunctionDescription(); + super(new FunctionMetadata( + new Signature( + ((TopLevelStdUDF) stdUDF).getFunctionName(), + getTypeVariableConstraintsForStdUdf(stdUDF), + ImmutableList.of(), + parseTypeSignature(stdUDF.getOutputParameterSignature(),ImmutableSet.of()), + stdUDF.getInputParameterSignatures().stream() + .map(typeSignature -> parseTypeSignature(typeSignature, ImmutableSet.of())) + .collect(Collectors.toList()), + false), + true, + Booleans.asList(stdUDF.getNullableArguments()).stream() + .map(FunctionArgumentDefinition::new) + .collect(Collectors.toList()), + false, + false, + ((TopLevelStdUDF) stdUDF).getFunctionDescription(), + FunctionKind.SCALAR)); } @VisibleForTesting @@ -84,21 +97,6 @@ protected long getRefreshIntervalMillis() { return TimeUnit.DAYS.toMillis(DEFAULT_REFRESH_INTERVAL_DAYS); } - @Override - public boolean isHidden() { - return false; - } - - @Override - public boolean isDeterministic() { - return false; - } - - @Override - public String getDescription() { - return _functionDescription; - } - @Override public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, Metadata metadata) { StdFactory stdFactory = new PrestoFactory(boundVariables, metadata); @@ -113,8 +111,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in boolean[] nullableArguments = stdUDF.getAndCheckNullableArguments(); return new ScalarFunctionImplementation(true, getNullConventionForArguments(nullableArguments), - getMethodHandle(stdUDF, metadata, boundVariables, nullableArguments, requiredFilesNextRefreshTime), - isDeterministic()); + getMethodHandle(stdUDF, metadata, boundVariables, nullableArguments, requiredFilesNextRefreshTime)); } private MethodHandle getMethodHandle(StdUDF stdUDF, Metadata metadata, BoundVariables boundVariables, @@ -277,7 +274,7 @@ private Type[] getPrestoTypes(List parameterSignatures, Metadata metadat } private Type getPrestoType(String parameterSignature, Metadata metadata, BoundVariables boundVariables) { - return metadata.getType(applyBoundVariables(TypeSignature.parseTypeSignature(parameterSignature), boundVariables)); + return metadata.getType(applyBoundVariables(parseTypeSignature(parameterSignature, ImmutableSet.of()), boundVariables)); } private Class[] getMethodHandleArgumentTypes(Type[] argTypes, boolean[] nullableArguments, diff --git a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoMap.java b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoMap.java index 4337819d..6344458d 100644 --- a/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoMap.java +++ b/transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoMap.java @@ -54,7 +54,7 @@ public PrestoMap(Type mapType, StdFactory stdFactory) { _stdFactory = stdFactory; _keyEqualsMethod = ((PrestoFactory) stdFactory).getScalarFunctionImplementation( - internalOperator(OperatorType.EQUAL, BooleanType.BOOLEAN, ImmutableList.of(_keyType, _keyType))) + ((PrestoFactory) stdFactory).resolveOperator(OperatorType.EQUAL, ImmutableList.of(_keyType, _keyType))) .getMethodHandle(); }