Upgrade to PrestoSQL 333

defaultEnvironment.gradle

@@ -10,8 +10,8 @@ subprojects {
       url "https://conjars.org/repo"
     }
   }
-  project.ext.setProperty('presto-version', '319')
-  project.ext.setProperty('airlift-slice-version', '0.33')
+  project.ext.setProperty('presto-version', '333')
+  project.ext.setProperty('airlift-slice-version', '0.38')
   project.ext.setProperty('spark-group', 'org.apache.spark')
   project.ext.setProperty('spark-version', '2.3.0')
 }

docs/transport-udfs-presto.patch

@@ -1,28 +1,5 @@
-diff --git a/presto-main/pom.xml b/presto-main/pom.xml
-index 83a20a23fe..02afa5310f 100644
---- a/presto-main/pom.xml
-+++ b/presto-main/pom.xml
-@@ -424,6 +424,18 @@
-                             <classpathScope>test</classpathScope>
-                         </configuration>
-                     </plugin>
-+                    <plugin>
-+                        <groupId>org.apache.maven.plugins</groupId>
-+                        <artifactId>maven-jar-plugin</artifactId>
-+                        <version>2.2</version>
-+                        <executions>
-+                            <execution>
-+                                <goals>
-+                                    <goal>test-jar</goal>
-+                                </goals>
-+                            </execution>
-+                        </executions>
-+                    </plugin>
-                 </plugins>
-             </build>
-         </profile>
 diff --git a/presto-main/src/main/java/io/prestosql/server/PluginManager.java b/presto-main/src/main/java/io/prestosql/server/PluginManager.java
-index f02ceeab03..88de943bed 100644
+index abcd001031..053c17aeed 100644
 --- a/presto-main/src/main/java/io/prestosql/server/PluginManager.java
 +++ b/presto-main/src/main/java/io/prestosql/server/PluginManager.java
 @@ -23,6 +23,7 @@ import io.prestosql.connector.ConnectorManager;
@@ -31,17 +8,17 @@ index f02ceeab03..88de943bed 100644
  import io.prestosql.metadata.MetadataManager;
 +import io.prestosql.metadata.SqlScalarFunction;
  import io.prestosql.security.AccessControlManager;
+ import io.prestosql.security.GroupProviderManager;
  import io.prestosql.server.security.PasswordAuthenticatorManager;
- import io.prestosql.spi.Plugin;
-@@ -52,6 +53,7 @@ import java.util.List;
- import java.util.ServiceLoader;
+@@ -54,6 +55,7 @@ import java.util.ServiceLoader;
  import java.util.Set;
  import java.util.concurrent.atomic.AtomicBoolean;
+ import java.util.function.Supplier;
 +import java.util.stream.Collectors;
 
  import static com.google.common.base.Preconditions.checkState;
  import static io.prestosql.metadata.FunctionExtractor.extractFunctions;
-@@ -62,8 +64,22 @@ import static java.util.Objects.requireNonNull;
+@@ -64,8 +66,22 @@ import static java.util.Objects.requireNonNull;
  @ThreadSafe
  public class PluginManager
  {
@@ -54,17 +31,17 @@ index f02ceeab03..88de943bed 100644
      private static final ImmutableList<String> SPI_PACKAGES = ImmutableList.<String>builder()
 +            // io.prestosql.metadata is required for SqlScalarFunction and FunctionRegistry classes
 +            .add("io.prestosql.metadata.")
-+            // io.prestosql.operator.scalar is required for ScalarFunctionImplementation
-+            .add("io.prestosql.operator.scalar.")
++            // io.prestosql.operator. is required for ScalarFunctionImplementation and TypeSignatureParser
++            .add("io.prestosql.operator.")
              .add("io.prestosql.spi.")
 +            // io.prestosql.type is required for TypeManager, and all supported types
 +            .add("io.prestosql.type.")
 +            // io.prestosql.util is required for Reflection
 +            .add("io.prestosql.util.")
              .add("com.fasterxml.jackson.annotation.")
              .add("io.airlift.slice.")
-             .add("io.airlift.units.")
-@@ -155,7 +171,21 @@ public class PluginManager
+             .add("org.openjdk.jol.")
+@@ -159,11 +175,22 @@ public class PluginManager
      {
          ServiceLoader<Plugin> serviceLoader = ServiceLoader.load(Plugin.class, pluginClassLoader);
          List<Plugin> plugins = ImmutableList.copyOf(serviceLoader);
@@ -74,32 +51,21 @@ index f02ceeab03..88de943bed 100644
 +                pluginClassLoader);
 +        List<SqlScalarFunction> sqlScalarFunctions = ImmutableList.copyOf(sqlScalarFunctionsServiceLoader);
 +
-+        checkState(!plugins.isEmpty() || !sqlScalarFunctions.isEmpty(), "No service providers of type %s or %s",
-+                Plugin.class.getName(), SqlScalarFunction.class.getName());
++        checkState(!plugins.isEmpty() || !sqlScalarFunctions.isEmpty(), "No service providers of type %s or %s", Plugin.class.getName(), SqlScalarFunction.class.getName());
 +
-+        installPlugins(plugins);
-+
-+        registerSqlScalarFunctions(sqlScalarFunctions);
-+    }
-+
-+    private void installPlugins(List<Plugin> plugins)
-+    {
          for (Plugin plugin : plugins) {
              log.info("Installing %s", plugin.getClass().getName());
-             installPlugin(plugin);
-@@ -215,6 +245,15 @@ public class PluginManager
+             installPlugin(plugin, pluginClassLoader::duplicate);
          }
-     }
-
-+    public void registerSqlScalarFunctions(List<SqlScalarFunction> sqlScalarFunctions)
-+    {
++
 +        for (SqlScalarFunction sqlScalarFunction : sqlScalarFunctions) {
-+            log.info("Registering function %s(%s)", sqlScalarFunction.getSignature().getName(), sqlScalarFunction.getSignature().getArgumentTypes().stream().map(e -> e.toString()).collect(
-+                    Collectors.joining(", ")));
++            log.info("Registering function %s(%s)",
++                    sqlScalarFunction.getFunctionMetadata().getSignature().getName(),
++                    sqlScalarFunction.getFunctionMetadata().getSignature().getArgumentTypes().stream()
++                            .map(e -> e.toString())
++                            .collect(Collectors.joining(", ")));
 +            metadataManager.addFunctions(ImmutableList.of(sqlScalarFunction));
 +        }
-+    }
-+
-     private URLClassLoader buildClassLoader(String plugin)
-             throws Exception
-     {
+     }
+
+     public void installPlugin(Plugin plugin, Supplier<ClassLoader> duplicatePluginClassLoaderFactory)

transportable-udfs-examples/build.gradle

@@ -33,8 +33,8 @@ subprojects {
       url "https://conjars.org/repo"
     }
   }
-  project.ext.setProperty('presto-version', '319')
-  project.ext.setProperty('airlift-slice-version', '0.33')
+  project.ext.setProperty('presto-version', '333')
+  project.ext.setProperty('airlift-slice-version', '0.38')
   project.ext.setProperty('spark-group', 'org.apache.spark')
   project.ext.setProperty('spark-version', '2.3.0')
 }

transportable-udfs-plugin/build.gradle

@@ -27,7 +27,7 @@ def writeVersionInfo = { file ->
   ant.propertyfile(file: file) {
     entry(key: "transport-version", value: version)
     entry(key: "hive-version", value: '1.2.2')
-    entry(key: "presto-version", value: '319')
+    entry(key: "presto-version", value: '333')
     entry(key: "spark-version", value: '2.3.0')
     entry(key: "scala-version", value: '2.11.8')
   }

transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/PrestoFactory.java

@@ -6,6 +6,7 @@
 package com.linkedin.transport.presto;
 
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
 import com.linkedin.transport.api.StdFactory;
 import com.linkedin.transport.api.data.StdArray;
 import com.linkedin.transport.api.data.StdBoolean;
@@ -25,18 +26,19 @@
 import io.airlift.slice.Slices;
 import io.prestosql.metadata.BoundVariables;
 import io.prestosql.metadata.Metadata;
-import io.prestosql.metadata.Signature;
+import io.prestosql.metadata.OperatorNotFoundException;
+import io.prestosql.metadata.ResolvedFunction;
 import io.prestosql.operator.scalar.ScalarFunctionImplementation;
+import io.prestosql.spi.function.OperatorType;
 import io.prestosql.spi.type.ArrayType;
 import io.prestosql.spi.type.MapType;
 import io.prestosql.spi.type.RowType;
 import io.prestosql.spi.type.Type;
-import io.prestosql.spi.type.TypeSignature;
 import java.util.List;
 import java.util.stream.Collectors;
 
 import static io.prestosql.metadata.SignatureBinder.*;
-
+import static io.prestosql.operator.TypeSignatureParser.parseTypeSignature;
 
 public class PrestoFactory implements StdFactory {
 
@@ -104,10 +106,14 @@ public StdStruct createStruct(StdType stdType) {
   @Override
   public StdType createStdType(String typeSignature) {
     return PrestoWrapper.createStdType(
-        metadata.getType(applyBoundVariables(TypeSignature.parseTypeSignature(typeSignature), boundVariables)));
+        metadata.getType(applyBoundVariables(parseTypeSignature(typeSignature, ImmutableSet.of()), boundVariables)));
+  }
+
+  public ScalarFunctionImplementation getScalarFunctionImplementation(ResolvedFunction resolvedFunction) {
+    return metadata.getScalarFunctionImplementation(resolvedFunction);
   }
 
-  public ScalarFunctionImplementation getScalarFunctionImplementation(Signature signature) {
-    return metadata.getScalarFunctionImplementation(signature);
+  public ResolvedFunction resolveOperator(OperatorType operatorType, List<? extends Type> argumentTypes) throws OperatorNotFoundException {
+    return metadata.resolveOperator(operatorType, argumentTypes);
   }
 }

transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/StdUdfWrapper.java

@@ -7,6 +7,8 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.primitives.Booleans;
 import com.linkedin.transport.api.StdFactory;
 import com.linkedin.transport.api.data.PlatformData;
 import com.linkedin.transport.api.data.StdData;
@@ -23,7 +25,9 @@
 import com.linkedin.transport.api.udf.TopLevelStdUDF;
 import com.linkedin.transport.typesystem.GenericTypeSignatureElement;
 import io.prestosql.metadata.BoundVariables;
+import io.prestosql.metadata.FunctionArgumentDefinition;
 import io.prestosql.metadata.FunctionKind;
+import io.prestosql.metadata.FunctionMetadata;
 import io.prestosql.metadata.Metadata;
 import io.prestosql.metadata.Signature;
 import io.prestosql.metadata.SqlScalarFunction;
@@ -32,7 +36,6 @@
 import io.prestosql.spi.classloader.ThreadContextClassLoader;
 import io.prestosql.spi.type.IntegerType;
 import io.prestosql.spi.type.Type;
-import io.prestosql.spi.type.TypeSignature;
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodType;
@@ -48,7 +51,7 @@
 
 import static io.prestosql.metadata.Signature.*;
 import static io.prestosql.metadata.SignatureBinder.*;
-import static io.prestosql.spi.type.TypeSignature.*;
+import static io.prestosql.operator.TypeSignatureParser.parseTypeSignature;
 import static io.prestosql.util.Reflection.*;
 
 // Suppressing argument naming convention for the evalInternal methods
@@ -57,16 +60,26 @@ public abstract class StdUdfWrapper extends SqlScalarFunction {
 
   private static final int DEFAULT_REFRESH_INTERVAL_DAYS = 1;
   private static final int JITTER_FACTOR = 50;  // to calculate jitter from delay
-  private String _functionDescription;
 
   protected StdUdfWrapper(StdUDF stdUDF) {
-    super(new Signature(((TopLevelStdUDF) stdUDF).getFunctionName(), FunctionKind.SCALAR,
-        getTypeVariableConstraintsForStdUdf(stdUDF), ImmutableList.of(),
-        parseTypeSignature(stdUDF.getOutputParameterSignature()), stdUDF.getInputParameterSignatures()
-        .stream()
-        .map(TypeSignature::parseTypeSignature)
-        .collect(Collectors.toList()), false));
-    _functionDescription = ((TopLevelStdUDF) stdUDF).getFunctionDescription();
+    super(new FunctionMetadata(
+            new Signature(
+                    ((TopLevelStdUDF) stdUDF).getFunctionName(),
+                    getTypeVariableConstraintsForStdUdf(stdUDF),
+                    ImmutableList.of(),
+                    parseTypeSignature(stdUDF.getOutputParameterSignature(),ImmutableSet.of()),
+                    stdUDF.getInputParameterSignatures().stream()
+                            .map(typeSignature -> parseTypeSignature(typeSignature, ImmutableSet.of()))
+                            .collect(Collectors.toList()),
+                    false),
+            true,
+            Booleans.asList(stdUDF.getNullableArguments()).stream()
+                    .map(FunctionArgumentDefinition::new)
+                    .collect(Collectors.toList()),
+            false,
+            false,
+            ((TopLevelStdUDF) stdUDF).getFunctionDescription(),
+            FunctionKind.SCALAR));
   }
 
   @VisibleForTesting
@@ -84,21 +97,6 @@ protected long getRefreshIntervalMillis() {
     return TimeUnit.DAYS.toMillis(DEFAULT_REFRESH_INTERVAL_DAYS);
   }
 
-  @Override
-  public boolean isHidden() {
-    return false;
-  }
-
-  @Override
-  public boolean isDeterministic() {
-    return false;
-  }
-
-  @Override
-  public String getDescription() {
-    return _functionDescription;
-  }
-
   @Override
   public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, Metadata metadata) {
     StdFactory stdFactory = new PrestoFactory(boundVariables, metadata);
@@ -113,8 +111,7 @@ public ScalarFunctionImplementation specialize(BoundVariables boundVariables, in
     boolean[] nullableArguments = stdUDF.getAndCheckNullableArguments();
 
     return new ScalarFunctionImplementation(true, getNullConventionForArguments(nullableArguments),
-        getMethodHandle(stdUDF, metadata, boundVariables, nullableArguments, requiredFilesNextRefreshTime),
-        isDeterministic());
+        getMethodHandle(stdUDF, metadata, boundVariables, nullableArguments, requiredFilesNextRefreshTime));
   }
 
   private MethodHandle getMethodHandle(StdUDF stdUDF, Metadata metadata, BoundVariables boundVariables,
@@ -277,7 +274,7 @@ private Type[] getPrestoTypes(List<String> parameterSignatures, Metadata metadat
   }
 
   private Type getPrestoType(String parameterSignature, Metadata metadata, BoundVariables boundVariables) {
-    return metadata.getType(applyBoundVariables(TypeSignature.parseTypeSignature(parameterSignature), boundVariables));
+    return metadata.getType(applyBoundVariables(parseTypeSignature(parameterSignature, ImmutableSet.of()), boundVariables));
   }
 
   private Class<?>[] getMethodHandleArgumentTypes(Type[] argTypes, boolean[] nullableArguments,

transportable-udfs-presto/src/main/java/com/linkedin/transport/presto/data/PrestoMap.java

@@ -54,7 +54,7 @@ public PrestoMap(Type mapType, StdFactory stdFactory) {
 
     _stdFactory = stdFactory;
     _keyEqualsMethod = ((PrestoFactory) stdFactory).getScalarFunctionImplementation(
-            internalOperator(OperatorType.EQUAL, BooleanType.BOOLEAN, ImmutableList.of(_keyType, _keyType)))
+            ((PrestoFactory) stdFactory).resolveOperator(OperatorType.EQUAL, ImmutableList.of(_keyType, _keyType)))
         .getMethodHandle();
   }