From 62ff286978badb2a65cd89738ddd1748544511ff Mon Sep 17 00:00:00 2001 From: Joan Goyeau Date: Tue, 25 Aug 2020 09:41:26 -0700 Subject: [PATCH] Ability to relocate/shade in assembly --- build.sc | 4 +- docs/pages/2 - Configuring Mill.md | 5 ++- main/src/modules/Assembly.scala | 29 +++++++++++--- main/src/modules/Jvm.scala | 2 +- .../hello-world-deps/core/src/Main.scala | 13 ++++++ scalalib/test/src/HelloWorldTests.scala | 40 +++++++++++++++++-- 6 files changed, 81 insertions(+), 12 deletions(-) create mode 100644 scalalib/test/resources/hello-world-deps/core/src/Main.scala diff --git a/build.sc b/build.sc index 3cbac66b5bf..a5854e46c7a 100755 --- a/build.sc +++ b/build.sc @@ -74,6 +74,7 @@ object Deps { val utest = ivy"com.lihaoyi::utest:0.7.4" val zinc = ivy"org.scala-sbt::zinc:1.4.0-M1" val bsp = ivy"ch.epfl.scala:bsp4j:2.0.0-M4" + val jarjarabrams = ivy"com.eed3si9n.jarjarabrams::jarjar-abrams-core:0.3.0" } trait MillPublishModule extends PublishModule{ @@ -167,7 +168,8 @@ object main extends MillModule { // Necessary so we can share the JNA classes throughout the build process Deps.jna, Deps.jnaPlatform, - Deps.coursier + Deps.coursier, + Deps.jarjarabrams ) def generatedSources = T { diff --git a/docs/pages/2 - Configuring Mill.md b/docs/pages/2 - Configuring Mill.md index 082c0aa9f3d..143657080b9 100644 --- a/docs/pages/2 - Configuring Mill.md +++ b/docs/pages/2 - Configuring Mill.md @@ -513,7 +513,7 @@ compilation output, but if there is more than one or the main class comes from some library you can explicitly specify which one to use. This also adds the main class to your `foo.jar` and `foo.assembly` jars. -## Merge/exclude files from assembly +## Merge/exclude/relocate files from assembly When you make a runnable jar of your project with `assembly` command, you may want to exclude some files from a final jar (like signature files, and manifest files from library jars), @@ -532,7 +532,8 @@ object foo extends ScalaModule { def assemblyRules = Seq( Rule.Append("application.conf"), // all application.conf files will be concatenated into single file Rule.AppendPattern(".*\\.conf"), // all *.conf files will be concatenated into single file - Rule.ExcludePattern("*.temp") // all *.temp files will be excluded from a final jar + Rule.ExcludePattern("*.temp"), // all *.temp files will be excluded from a final jar + Rule.Relocate("shapeless.**", "shade.shapless.@1") // the `shapeless` package will be shaded under the `shade` package ) } ``` diff --git a/main/src/modules/Assembly.scala b/main/src/modules/Assembly.scala index 29472868895..7e1b9794607 100644 --- a/main/src/modules/Assembly.scala +++ b/main/src/modules/Assembly.scala @@ -1,11 +1,13 @@ package mill.modules -import java.io.InputStream +import com.eed3si9n.jarjarabrams.{ShadePattern, Shader} +import java.io.{ByteArrayInputStream, InputStream} import java.util.jar.JarFile import java.util.regex.Pattern import mill.Agg import os.Generator import scala.collection.JavaConverters._ +import scala.tools.nsc.io.Streamable object Assembly { @@ -30,6 +32,8 @@ object Assembly { case class Exclude(path: String) extends Rule + case class Relocate(from: String, to: String) extends Rule + object ExcludePattern { def apply(pattern: String): ExcludePattern = ExcludePattern(Pattern.compile(pattern)) } @@ -74,20 +78,35 @@ object Assembly { } } - def loadClasspath( - inputPaths: Agg[os.Path] + def loadShadedClasspath( + inputPaths: Agg[os.Path], + assemblyRules: Seq[Assembly.Rule] ): Generator[(String, UnopenedInputStream)] = { + val shadeRules = assemblyRules.collect { + case Rule.Relocate(from, to) => ShadePattern.Rename(List(from -> to)).inAll + } + val shader = + if (shadeRules.isEmpty) (name: String, inputStream: UnopenedInputStream) => Some(name -> inputStream) + else { + val shader = Shader.bytecodeShader(shadeRules, verbose = false) + (name: String, inputStream: UnopenedInputStream) => + shader(Streamable.bytes(inputStream()), name).map { + case (bytes, name) => + name -> (() => new ByteArrayInputStream(bytes) { override def close(): Unit = inputStream().close() }) + } + } + Generator.from(inputPaths).filter(os.exists).flatMap { path => if (os.isFile(path)) { val jarFile = new JarFile(path.toIO) Generator.from(jarFile.entries().asScala.filterNot(_.isDirectory)) - .map(entry => entry.getName -> (() => jarFile.getInputStream(entry))) + .flatMap(entry => shader(entry.getName, () => jarFile.getInputStream(entry))) } else { os.walk .stream(path) .filter(os.isFile) - .map(subPath => subPath.relativeTo(path).toString -> (() => os.read.inputStream(subPath))) + .flatMap(subPath => shader(subPath.relativeTo(path).toString, () => os.read.inputStream(subPath))) } } } diff --git a/main/src/modules/Jvm.scala b/main/src/modules/Jvm.scala index 00ef83fd953..f2ce892cb80 100644 --- a/main/src/modules/Jvm.scala +++ b/main/src/modules/Jvm.scala @@ -286,7 +286,7 @@ object Jvm { manifest.build.write(manifestOut) manifestOut.close() - val mappings = Assembly.loadClasspath(inputPaths) + val mappings = Assembly.loadShadedClasspath(inputPaths, assemblyRules) Assembly.groupAssemblyEntries(mappings, assemblyRules).foreach { case (mapping, entry) => val path = zipFs.getPath(mapping).toAbsolutePath diff --git a/scalalib/test/resources/hello-world-deps/core/src/Main.scala b/scalalib/test/resources/hello-world-deps/core/src/Main.scala new file mode 100644 index 00000000000..4ac7cc3e330 --- /dev/null +++ b/scalalib/test/resources/hello-world-deps/core/src/Main.scala @@ -0,0 +1,13 @@ +import akka.http.scaladsl.model.{ContentTypes, HttpEntity} +import akka.http.scaladsl.server.Directives._ + +object Main extends App { + val route = + path("hello") { + get { + complete(HttpEntity(ContentTypes.`text/html(UTF-8)`, "

Say hello to akka-http

")) + } + } + + println(route) +} diff --git a/scalalib/test/src/HelloWorldTests.scala b/scalalib/test/src/HelloWorldTests.scala index e91b24b8be3..4b509465955 100644 --- a/scalalib/test/src/HelloWorldTests.scala +++ b/scalalib/test/src/HelloWorldTests.scala @@ -87,6 +87,14 @@ object HelloWorldTests extends TestSuite { } } + object HelloWorldAkkaHttpRelocate extends HelloBase { + object core extends HelloWorldModuleWithMain { + def ivyDeps = akkaHttpDeps + + def assemblyRules = Seq(Assembly.Rule.Relocate("akka.**", "shaded.akka.@1")) + } + } + object HelloWorldAkkaHttpNoRules extends HelloBase { object core extends HelloWorldModuleWithMain { def ivyDeps = akkaHttpDeps @@ -684,9 +692,7 @@ object HelloWorldTests extends TestSuite { referenceContent.contains("Akka Stream Reference Config File"), // our application config is present too referenceContent.contains("My application Reference Config File"), - referenceContent.contains( - """akka.http.client.user-agent-header="hello-world-client"""" - ) + referenceContent.contains("""akka.http.client.user-agent-header="hello-world-client"""") ) } @@ -767,6 +773,34 @@ object HelloWorldTests extends TestSuite { resourcePath = helloWorldMultiResourcePath ) + def checkRelocate[M <: TestUtil.BaseModule](module: M, + target: Target[PathRef], + resourcePath: os.Path = resourcePath + ) = + workspaceTest(module, resourcePath) { eval => + val Right((result, _)) = eval.apply(target) + + val jarFile = new JarFile(result.path.toIO) + + assert(!jarEntries(jarFile).contains("akka/http/scaladsl/model/HttpEntity.class")) + assert(jarEntries(jarFile).contains("shaded/akka/http/scaladsl/model/HttpEntity.class")) + } + + 'relocate - { + 'withDeps - checkRelocate( + HelloWorldAkkaHttpRelocate, + HelloWorldAkkaHttpRelocate.core.assembly + ) + + 'run - workspaceTest( + HelloWorldAkkaHttpRelocate, + resourcePath = os.pwd / 'scalalib / 'test / 'resources / "hello-world-deps" + ) { eval => + val Right((_, evalCount)) = eval.apply(HelloWorldAkkaHttpRelocate.core.runMain("Main")) + assert(evalCount > 0) + } + } + 'writeDownstreamWhenNoRule - { 'withDeps - workspaceTest(HelloWorldAkkaHttpNoRules) { eval => val Right((result, _)) = eval.apply(HelloWorldAkkaHttpNoRules.core.assembly)