From 4af5057d067880c1dca92bca8f4392fce2d0156d Mon Sep 17 00:00:00 2001 From: Shardul Mahadik Date: Thu, 6 Aug 2020 22:48:45 -0700 Subject: [PATCH 1/2] Publish Presto thin jar which allows consumers to control dependency graph --- .../linkedin/transport/plugin/Defaults.java | 9 +++-- .../linkedin/transport/plugin/Platform.java | 6 +-- .../transport/plugin/TransportPlugin.java | 5 ++- .../packaging/DistributionPackaging.java | 2 +- .../plugin/packaging/ThinJarPackaging.java | 37 +++++++++++++++++++ 5 files changed, 50 insertions(+), 9 deletions(-) create mode 100644 transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/ThinJarPackaging.java diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java index ee96e18d..64a967f4 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java @@ -11,6 +11,7 @@ import com.linkedin.transport.codegen.SparkWrapperGenerator; import com.linkedin.transport.plugin.packaging.DistributionPackaging; import com.linkedin.transport.plugin.packaging.ShadedJarPackaging; +import com.linkedin.transport.plugin.packaging.ThinJarPackaging; import java.io.IOException; import java.io.InputStream; import java.util.List; @@ -72,7 +73,7 @@ private static Properties loadDefaultVersions() { // converters drop dependencies with classifiers, so we apply this dependency explicitly getDependencyConfiguration(RUNTIME_ONLY, "io.prestosql:presto-main", "presto", "tests") ), - new DistributionPackaging()), + ImmutableList.of(new ThinJarPackaging(), new DistributionPackaging())), new Platform( "hive", Language.JAVA, @@ -85,7 +86,7 @@ private static Properties loadDefaultVersions() { getDependencyConfiguration(RUNTIME_ONLY, "com.linkedin.transport:transportable-udfs-test-hive", "transport") ), - new ShadedJarPackaging(ImmutableList.of("org.apache.hadoop", "org.apache.hive"), null)), + ImmutableList.of(new ShadedJarPackaging(ImmutableList.of("org.apache.hadoop", "org.apache.hive"), null))), new Platform( "spark", Language.SCALA, @@ -99,9 +100,9 @@ private static Properties loadDefaultVersions() { getDependencyConfiguration(RUNTIME_ONLY, "com.linkedin.transport:transportable-udfs-test-spark", "transport") ), - new ShadedJarPackaging( + ImmutableList.of(new ShadedJarPackaging( ImmutableList.of("org.apache.hadoop", "org.apache.spark"), - ImmutableList.of("com.linkedin.transport.spark.**")) + ImmutableList.of("com.linkedin.transport.spark.**"))) ) ); diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Platform.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Platform.java index fed24365..b3d87679 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Platform.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Platform.java @@ -20,11 +20,11 @@ public class Platform { private final Class _wrapperGeneratorClass; private final List _defaultWrapperDependencyConfigurations; private final List _defaultTestDependencyConfigurations; - private final Packaging _packaging; + private final List _packaging; public Platform(String name, Language language, Class wrapperGeneratorClass, List defaultWrapperDependencyConfigurations, - List defaultTestDependencyConfigurations, Packaging packaging) { + List defaultTestDependencyConfigurations, List packaging) { _name = name; _language = language; _wrapperGeneratorClass = wrapperGeneratorClass; @@ -53,7 +53,7 @@ public List getDefaultTestDependencyConfigurations() { return _defaultTestDependencyConfigurations; } - public Packaging getPackaging() { + public List getPackaging() { return _packaging; } } diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java index 55c5778b..2f8e2984 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java @@ -12,6 +12,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; +import java.util.stream.Collectors; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Task; @@ -203,7 +204,9 @@ private TaskProvider configureGenerateWrappersTask(Project */ private List> configurePackagingTasks(Project project, Platform platform, SourceSet sourceSet, SourceSet mainSourceSet) { - return platform.getPackaging().configurePackagingTasks(project, platform, sourceSet, mainSourceSet); + return platform.getPackaging().stream() + .flatMap(p -> p.configurePackagingTasks(project, platform, sourceSet, mainSourceSet).stream()) + .collect(Collectors.toList()); } /** diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/DistributionPackaging.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/DistributionPackaging.java index 44ef8b55..76e3a41f 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/DistributionPackaging.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/DistributionPackaging.java @@ -73,7 +73,7 @@ private TaskProvider createThinJarTask(Project project, SourceSet sourceSet } */ - return project.getTasks().register(sourceSet.getTaskName(null, "thinJar"), Jar.class, task -> { + return project.getTasks().register(sourceSet.getTaskName(null, "distThinJar"), Jar.class, task -> { task.dependsOn(project.getTasks().named(sourceSet.getClassesTaskName())); task.setDescription("Assembles a thin jar archive containing the " + platformName + " classes to be included in the distribution"); diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/ThinJarPackaging.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/ThinJarPackaging.java new file mode 100644 index 00000000..b03a3443 --- /dev/null +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/ThinJarPackaging.java @@ -0,0 +1,37 @@ +/** + * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.plugin.packaging; + +import com.google.common.collect.ImmutableList; +import com.linkedin.transport.plugin.Platform; +import java.util.List; +import org.gradle.api.Project; +import org.gradle.api.Task; +import org.gradle.api.artifacts.Dependency; +import org.gradle.api.tasks.SourceSet; +import org.gradle.api.tasks.TaskProvider; +import org.gradle.api.tasks.bundling.Jar; + + +public class ThinJarPackaging implements Packaging { + + @Override + public List> configurePackagingTasks(Project project, Platform platform, + SourceSet platformSourceSet, SourceSet mainSourceSet) { + TaskProvider thinJarTask = + project.getTasks().register(platformSourceSet.getTaskName(null, "thinJar"), Jar.class, task -> { + task.dependsOn(project.getTasks().named(platformSourceSet.getClassesTaskName())); + task.setDescription("Assembles a thin jar archive containing the " + platform.getName() + + " classes to be included in the distribution"); + task.setClassifier(platform.getName() + "Thin"); + task.from(platformSourceSet.getOutput()); + task.from(platformSourceSet.getResources()); + }); + + project.getArtifacts().add(Dependency.ARCHIVES_CONFIGURATION, thinJarTask); + return ImmutableList.of(thinJarTask); + } +} From fe2c57207400b63eeb9900aad2923f08ddb296e3 Mon Sep 17 00:00:00 2001 From: Shardul Mahadik Date: Fri, 7 Aug 2020 11:26:05 -0700 Subject: [PATCH 2/2] Address PR comments --- .../plugin/packaging/DistributionPackaging.java | 6 +++--- .../plugin/packaging/ThinJarPackaging.java | 13 ++++++++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/DistributionPackaging.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/DistributionPackaging.java index 76e3a41f..13de71db 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/DistributionPackaging.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/DistributionPackaging.java @@ -66,8 +66,8 @@ public List> configurePackagingTasks(Project projec */ private TaskProvider createThinJarTask(Project project, SourceSet sourceSet, String platformName) { /* - task ThinJar(type: Jar, dependsOn: prestoClasses) { - classifier 'platformName' + task DistThinJar(type: Jar, dependsOn: prestoClasses) { + classifier '-dist-thin' from sourceSets..output from sourceSets..resources } @@ -77,7 +77,7 @@ private TaskProvider createThinJarTask(Project project, SourceSet sourceSet task.dependsOn(project.getTasks().named(sourceSet.getClassesTaskName())); task.setDescription("Assembles a thin jar archive containing the " + platformName + " classes to be included in the distribution"); - task.setClassifier(platformName + "Thin"); + task.setClassifier(platformName + "-dist-thin"); task.from(sourceSet.getOutput()); task.from(sourceSet.getResources()); }); diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/ThinJarPackaging.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/ThinJarPackaging.java index b03a3443..7367733c 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/ThinJarPackaging.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/packaging/ThinJarPackaging.java @@ -16,17 +16,28 @@ import org.gradle.api.tasks.bundling.Jar; +/** + * A {@link Packaging} class which generates a Jar containing the autogenerated code for the platform + */ public class ThinJarPackaging implements Packaging { @Override public List> configurePackagingTasks(Project project, Platform platform, SourceSet platformSourceSet, SourceSet mainSourceSet) { + /* + task ThinJar(type: Jar, dependsOn: prestoClasses) { + classifier '-thin' + from sourceSets..output + from sourceSets..resources + } + */ + TaskProvider thinJarTask = project.getTasks().register(platformSourceSet.getTaskName(null, "thinJar"), Jar.class, task -> { task.dependsOn(project.getTasks().named(platformSourceSet.getClassesTaskName())); task.setDescription("Assembles a thin jar archive containing the " + platform.getName() + " classes to be included in the distribution"); - task.setClassifier(platform.getName() + "Thin"); + task.setClassifier(platform.getName() + "-thin"); task.from(platformSourceSet.getOutput()); task.from(platformSourceSet.getResources()); });