From 62336680a5c13ffbf5c1acd8a5375a44dc3c0837 Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Thu, 20 Jun 2024 14:07:18 +0200 Subject: [PATCH 1/6] Improve Mill plugin classpath exclusions --- build.sc | 24 +++- .../feature/plugin-classpath/repo/build.sc | 9 ++ .../test/src/MillPluginClasspathTest.scala | 131 ++++++++++++++++++ .../integration/IntegrationTestSuite.scala | 11 ++ 4 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 integration/feature/plugin-classpath/repo/build.sc create mode 100644 integration/feature/plugin-classpath/test/src/MillPluginClasspathTest.scala diff --git a/build.sc b/build.sc index b9d8d6c6228..8bd668ed571 100644 --- a/build.sc +++ b/build.sc @@ -201,7 +201,7 @@ object Deps { ivy"com.google.protobuf:protobuf-java:3.25.3", ivy"com.google.guava:guava:33.2.1-jre", ivy"org.yaml:snakeyaml:2.2", - ivy"org.apache.commons:commons-compress:[1.26.0,)" + ivy"org.apache.commons:commons-compress:1.26.0" ) /** Used in tests. */ @@ -588,8 +588,25 @@ object main extends MillStableScalaModule with BuildInfo { BuildInfo.Value("millBinPlatform", millBinPlatform(), "Mill binary platform version."), BuildInfo.Value( "millEmbeddedDeps", - T.traverse(dev.moduleDeps)(_.publishSelfDependency)() - .map(artifact => s"${artifact.group}:${artifact.id}:${artifact.version}") + ( + T.traverse( + dev.transitiveModuleDeps.collect { case m: PublishModule => m } + )( + _.publishSelfDependency + )() + .map(artifact => s"${artifact.group}:${artifact.id}:${artifact.version}") ++ + Lib.resolveDependenciesMetadata( + repositories = dev.repositoriesTask(), + dev.transitiveIvyDeps(), + Some(dev.mapDependencies()), + dev.resolutionCustomizer(), + Some(T.ctx()), + dev.coursierCacheCustomizer() + )._2.minDependencies.toSeq + .map(d => s"${d.module.organization.value}:${d.module.name.value}:${d.version}") + ) +// T.traverse(dev.moduleDeps)(_.publishSelfDependency)() +// .map(artifact => s"${artifact.group}:${artifact.id}:${artifact.version}") .mkString(","), "Dependency artifacts embedded in mill assembly by default." ), @@ -1009,7 +1026,6 @@ object contrib extends Module { def ivyDeps = Agg(Deps.requests) } - object sonatypecentral extends ContribModule { def compileModuleDeps = Seq(scalalib) def ivyDeps = Agg(Deps.sonatypeCentralClient) diff --git a/integration/feature/plugin-classpath/repo/build.sc b/integration/feature/plugin-classpath/repo/build.sc new file mode 100644 index 00000000000..c6be03a3bcc --- /dev/null +++ b/integration/feature/plugin-classpath/repo/build.sc @@ -0,0 +1,9 @@ +// This plugin brings a incompatible transitive version of Mill +import $ivy.`com.disneystreaming.smithy4s::smithy4s-mill-codegen-plugin::0.18.22` + +import mill._ +import mill.scalalib._ + +object root extends RootModule with ScalaModule { + def scalaVersion = "3.4.2" +} diff --git a/integration/feature/plugin-classpath/test/src/MillPluginClasspathTest.scala b/integration/feature/plugin-classpath/test/src/MillPluginClasspathTest.scala new file mode 100644 index 00000000000..c253b3d0d0f --- /dev/null +++ b/integration/feature/plugin-classpath/test/src/MillPluginClasspathTest.scala @@ -0,0 +1,131 @@ +package mill.integration + +import utest._ + +object MillPluginClasspathTest extends IntegrationTestSuite { + initWorkspace() + + val embeddedModules = Seq( + ("com.lihaoyi", "mill-dev_2.13"), + ("com.lihaoyi", "mill-main-client"), + ("com.lihaoyi", "mill-main-api_2.13"), + ("com.lihaoyi", "mill-main-util_2.13"), + ("com.lihaoyi", "mill-main-codesig_2.13"), + ("com.lihaoyi", "mill-runner-linenumbers_2.13"), + ("com.lihaoyi", "mill-bsp_2.13"), + ("com.lihaoyi", "mill-scalanativelib-worker-api_2.13"), + ("com.lihaoyi", "mill-testrunner-entrypoint"), + ("com.lihaoyi", "mill-scalalib-api_2.13"), + ("com.lihaoyi", "mill-testrunner_2.13"), + ("com.lihaoyi", "mill-main-define_2.13"), + ("com.lihaoyi", "mill-main-resolve_2.13"), + ("com.lihaoyi", "mill-main-eval_2.13"), + ("com.lihaoyi", "mill-main_2.13"), + ("com.lihaoyi", "mill-scalalib_2.13"), + ("com.lihaoyi", "mill-scalanativelib_2.13"), + ("com.lihaoyi", "mill-scalajslib-worker-api_2.13"), + ("com.lihaoyi", "mill-scalajslib_2.13"), + ("com.lihaoyi", "mill-runner_2.13"), + ("com.lihaoyi", "mill-idea_2.13") +// ("com.lihaoyi", "mainargs_2.13"), +// ("org.codehaus.plexus", "plexus-utils"), +// ("com.lihaoyi", "upack_2.13"), +// ("org.codehaus.plexus", "plexus-container-default"), +// ("com.lihaoyi", "sourcecode_2.13"), +// ("org.virtuslab.scala-cli", "config_2.13"), +// ("com.lihaoyi", "fastparse_2.13"), +// ("com.eed3si9n.jarjarabrams", "jarjar-abrams-core_2.13"), +// ("com.kohlschutter.junixsocket", "junixsocket-common"), +// ("com.lihaoyi", "requests_2.13"), +// ("org.scala-lang.modules", "scala-xml_2.13"), +// ("org.codehaus.plexus", "plexus-archiver"), +// ("io.get-coursier", "coursier-cache_2.13"), +// ("com.eed3si9n.jarjar", "jarjar"), +// ("com.lihaoyi", "ujson_2.13"), +// ("org.apache.xbean", "xbean-reflect"), +// ("com.lihaoyi", "upickle-implicits_2.13"), +// ("org.scalameta", "scalafmt-dynamic_2.13"), +// ("org.apache.commons", "commons-lang3"), +// ("org.scalameta", "scalafmt-interfaces"), +// ("com.lihaoyi", "upickle_2.13"), +// ("io.get-coursier", "coursier-proxy-setup"), +// ("net.java.dev.jna", "jna-platform"), +// ("com.lihaoyi", "scalaparse_2.13"), +// ("com.github.luben", "zstd-jni"), +// ("org.apache.commons", "commons-compress"), +// ("com.lihaoyi", "fansi_2.13"), +// ("org.codehaus.plexus", "plexus-io"), +// ("com.lihaoyi", "os-lib_2.13"), +// ("org.scala-lang.modules", "scala-collection-compat_2.13"), +// ("io.get-coursier", "coursier_2.13"), +// ("org.tukaani", "xz"), +// ("net.java.dev.jna", "jna"), +// ("com.kohlschutter.junixsocket", "junixsocket-native-common"), +// ("com.lihaoyi", "pprint_2.13"), +// ("com.lihaoyi", "upickle-core_2.13"), +// ("io.get-coursier.jniutils", "windows-jni-utils"), +// ("javax.inject", "javax.inject"), +// ("io.github.java-diff-utils", "java-diff-utils"), +// ("io.get-coursier", "interface"), +// ("org.scala-lang", "scala-compiler"), +// ("org.scala-lang", "scala-library"), +// ("io.get-coursier", "coursier-core_2.13"), +// ("org.iq80.snappy", "snappy"), +// ("org.fusesource.jansi", "jansi"), +// ("org.jline", "jline"), +// ("io.github.alexarchambault", "concurrent-reference-hash-map"), +// ("commons-io", "commons-io"), +// ("io.get-coursier", "coursier-util_2.13"), +// ("org.scala-sbt", "test-interface"), +// ("org.ow2.asm", "asm-tree"), +// ("org.ow2.asm", "asm-commons"), +// ("com.github.plokhotnyuk.jsoniter-scala", "jsoniter-scala-core_2.13"), +// ("org.scala-lang", "scala-reflect"), +// ("com.kohlschutter.junixsocket", "junixsocket-core"), +// ("com.lihaoyi", "geny_2.13"), +// ("com.typesafe", "config"), +// ("org.codehaus.plexus", "plexus-classworlds"), +// ("org.slf4j", "slf4j-api"), +// ("io.github.alexarchambault.windows-ansi", "windows-ansi"), +// ("org.ow2.asm", "asm"), +// ("com.lihaoyi", "mill-moduledefs_2.13") + ) + + val tests: Tests = Tests { + test("exclusions") - { + val res1 = eval("--meta-level", "1", "resolveDepsExclusions") + assert(res1) + + val exclusions = metaValue[Seq[(String, String)]]("mill-build.resolveDepsExclusions") + + // pprint.pprintln(exclusions) + val expectedExclusions = embeddedModules + + val diff = expectedExclusions.toSet.diff(exclusions.toSet) + assert(diff.isEmpty) + + } + test("runClasspath") - { + // We expect Mill core transitive dependencies to be filtered out + val res1 = eval("--meta-level", "1", "runClasspath") + assert(res1) + + val runClasspath = metaValue[Seq[String]]("mill-build.runClasspath") +// pprint.pprintln(runClasspath) + + val unexpectedArtifacts = embeddedModules.map { + case (o, n) => s"${o.replaceAll("[.]", "/")}/${n}" + } +// pprint.pprintln(unexpectedArtifacts) + + val unexpected = unexpectedArtifacts.flatMap { a => + runClasspath.find(p => p.toString.contains(a)).map((a, _)) + }.toMap + assert(unexpected.isEmpty) + + val expected = Seq("com/disneystreaming/smithy4s/smithy4s-mill-codegen-plugin_mill0.11_2.13") + assert(expected.forall(a => runClasspath.exists(p => p.toString().contains(a)))) + } + + } +} diff --git a/integration/src/mill/integration/IntegrationTestSuite.scala b/integration/src/mill/integration/IntegrationTestSuite.scala index 90d55f9e4d0..9c9cddd1997 100644 --- a/integration/src/mill/integration/IntegrationTestSuite.scala +++ b/integration/src/mill/integration/IntegrationTestSuite.scala @@ -1,5 +1,6 @@ package mill.integration +import mill.eval.Evaluator import mill.resolve.SelectMode import mill.runner.RunnerState import os.{Path, Shellable} @@ -91,6 +92,16 @@ abstract class IntegrationTestSuite extends TestSuite { os.read(wd / "out" / segments.init / s"${segments.last}.json") } + def metaCached(selector: String): Evaluator.Cached = { + val data = meta(selector) + upickle.default.read[Evaluator.Cached](data) + } + + def metaValue[T: upickle.default.Reader](selector: String): T = { + val cached = metaCached(selector) + upickle.default.read[T](cached.value) + } + def initWorkspace(): Path = { os.remove.all(workspacePath) os.makeDir.all(workspacePath / os.up) From dc7771581641b82e7d99ed4b069458513394487b Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Thu, 20 Jun 2024 14:19:23 +0200 Subject: [PATCH 2/6] . --- integration/feature/plugin-classpath/repo/build.sc | 1 + 1 file changed, 1 insertion(+) diff --git a/integration/feature/plugin-classpath/repo/build.sc b/integration/feature/plugin-classpath/repo/build.sc index c6be03a3bcc..1d90419b0b9 100644 --- a/integration/feature/plugin-classpath/repo/build.sc +++ b/integration/feature/plugin-classpath/repo/build.sc @@ -1,4 +1,5 @@ // This plugin brings a incompatible transitive version of Mill +// See https://github.com/com-lihaoyi/mill/issues/3207 import $ivy.`com.disneystreaming.smithy4s::smithy4s-mill-codegen-plugin::0.18.22` import mill._ From 8ea7390807a4af2b69bdb57970cb2b57fa305e72 Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Thu, 20 Jun 2024 15:10:57 +0200 Subject: [PATCH 3/6] Cleanup idea test case, why this failed now? --- .../repo/hello-idea/idea/libraries/scala_SDK_2_13_14.xml | 3 --- integration/feature/gen-idea/test/src/GenIdeaUtils.scala | 7 ++++--- .../test/src/MillPluginClasspathTest.scala | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/integration/feature/gen-idea/repo/hello-idea/idea/libraries/scala_SDK_2_13_14.xml b/integration/feature/gen-idea/repo/hello-idea/idea/libraries/scala_SDK_2_13_14.xml index 69833688c91..39cfc936520 100644 --- a/integration/feature/gen-idea/repo/hello-idea/idea/libraries/scala_SDK_2_13_14.xml +++ b/integration/feature/gen-idea/repo/hello-idea/idea/libraries/scala_SDK_2_13_14.xml @@ -3,9 +3,6 @@ Scala_2_13 - - - diff --git a/integration/feature/gen-idea/test/src/GenIdeaUtils.scala b/integration/feature/gen-idea/test/src/GenIdeaUtils.scala index 8cfac26a3e8..c3f8c56911a 100644 --- a/integration/feature/gen-idea/test/src/GenIdeaUtils.scala +++ b/integration/feature/gen-idea/test/src/GenIdeaUtils.scala @@ -23,7 +23,8 @@ object GenIdeaUtils { assertPartialContentMatches( found = actualResourceString, - expected = expectedResourceString + expected = expectedResourceString, + resource.toString() ) } println( @@ -33,9 +34,9 @@ object GenIdeaUtils { check.get } - def assertPartialContentMatches(found: String, expected: String): Unit = { + def assertPartialContentMatches(found: String, expected: String, context: String = ""): Unit = { if (!expected.contains(ignoreString)) { - assert(found == expected) + assert(context != null && found == expected) } val pattern = diff --git a/integration/feature/plugin-classpath/test/src/MillPluginClasspathTest.scala b/integration/feature/plugin-classpath/test/src/MillPluginClasspathTest.scala index c253b3d0d0f..8ccc71288a2 100644 --- a/integration/feature/plugin-classpath/test/src/MillPluginClasspathTest.scala +++ b/integration/feature/plugin-classpath/test/src/MillPluginClasspathTest.scala @@ -5,7 +5,7 @@ import utest._ object MillPluginClasspathTest extends IntegrationTestSuite { initWorkspace() - val embeddedModules = Seq( + val embeddedModules: Seq[(String, String)] = Seq( ("com.lihaoyi", "mill-dev_2.13"), ("com.lihaoyi", "mill-main-client"), ("com.lihaoyi", "mill-main-api_2.13"), From 72cac03be4ace1d4be831d28e25ba76b743122d2 Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Thu, 20 Jun 2024 16:30:37 +0200 Subject: [PATCH 4/6] Don't include mill-dev --- build.sc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sc b/build.sc index 8bd668ed571..18ba7b111e7 100644 --- a/build.sc +++ b/build.sc @@ -590,7 +590,7 @@ object main extends MillStableScalaModule with BuildInfo { "millEmbeddedDeps", ( T.traverse( - dev.transitiveModuleDeps.collect { case m: PublishModule => m } + dev.recursiveModuleDeps.collect { case m: PublishModule => m } )( _.publishSelfDependency )() From ecaefd13703253c5da2dba68f962cda511406318 Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Thu, 20 Jun 2024 16:40:18 +0200 Subject: [PATCH 5/6] . --- .../plugin-classpath/test/src/MillPluginClasspathTest.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/integration/feature/plugin-classpath/test/src/MillPluginClasspathTest.scala b/integration/feature/plugin-classpath/test/src/MillPluginClasspathTest.scala index 8ccc71288a2..f2426fb33fb 100644 --- a/integration/feature/plugin-classpath/test/src/MillPluginClasspathTest.scala +++ b/integration/feature/plugin-classpath/test/src/MillPluginClasspathTest.scala @@ -6,7 +6,6 @@ object MillPluginClasspathTest extends IntegrationTestSuite { initWorkspace() val embeddedModules: Seq[(String, String)] = Seq( - ("com.lihaoyi", "mill-dev_2.13"), ("com.lihaoyi", "mill-main-client"), ("com.lihaoyi", "mill-main-api_2.13"), ("com.lihaoyi", "mill-main-util_2.13"), From 461fd6b29dcf5965a1e9a1d5f086cd48d5d9de5c Mon Sep 17 00:00:00 2001 From: Tobias Roeser Date: Thu, 20 Jun 2024 22:14:06 +0200 Subject: [PATCH 6/6] cleanup comments --- .../test/src/MillPluginClasspathTest.scala | 66 ------------------- 1 file changed, 66 deletions(-) diff --git a/integration/feature/plugin-classpath/test/src/MillPluginClasspathTest.scala b/integration/feature/plugin-classpath/test/src/MillPluginClasspathTest.scala index f2426fb33fb..5e143db2d89 100644 --- a/integration/feature/plugin-classpath/test/src/MillPluginClasspathTest.scala +++ b/integration/feature/plugin-classpath/test/src/MillPluginClasspathTest.scala @@ -26,68 +26,6 @@ object MillPluginClasspathTest extends IntegrationTestSuite { ("com.lihaoyi", "mill-scalajslib_2.13"), ("com.lihaoyi", "mill-runner_2.13"), ("com.lihaoyi", "mill-idea_2.13") -// ("com.lihaoyi", "mainargs_2.13"), -// ("org.codehaus.plexus", "plexus-utils"), -// ("com.lihaoyi", "upack_2.13"), -// ("org.codehaus.plexus", "plexus-container-default"), -// ("com.lihaoyi", "sourcecode_2.13"), -// ("org.virtuslab.scala-cli", "config_2.13"), -// ("com.lihaoyi", "fastparse_2.13"), -// ("com.eed3si9n.jarjarabrams", "jarjar-abrams-core_2.13"), -// ("com.kohlschutter.junixsocket", "junixsocket-common"), -// ("com.lihaoyi", "requests_2.13"), -// ("org.scala-lang.modules", "scala-xml_2.13"), -// ("org.codehaus.plexus", "plexus-archiver"), -// ("io.get-coursier", "coursier-cache_2.13"), -// ("com.eed3si9n.jarjar", "jarjar"), -// ("com.lihaoyi", "ujson_2.13"), -// ("org.apache.xbean", "xbean-reflect"), -// ("com.lihaoyi", "upickle-implicits_2.13"), -// ("org.scalameta", "scalafmt-dynamic_2.13"), -// ("org.apache.commons", "commons-lang3"), -// ("org.scalameta", "scalafmt-interfaces"), -// ("com.lihaoyi", "upickle_2.13"), -// ("io.get-coursier", "coursier-proxy-setup"), -// ("net.java.dev.jna", "jna-platform"), -// ("com.lihaoyi", "scalaparse_2.13"), -// ("com.github.luben", "zstd-jni"), -// ("org.apache.commons", "commons-compress"), -// ("com.lihaoyi", "fansi_2.13"), -// ("org.codehaus.plexus", "plexus-io"), -// ("com.lihaoyi", "os-lib_2.13"), -// ("org.scala-lang.modules", "scala-collection-compat_2.13"), -// ("io.get-coursier", "coursier_2.13"), -// ("org.tukaani", "xz"), -// ("net.java.dev.jna", "jna"), -// ("com.kohlschutter.junixsocket", "junixsocket-native-common"), -// ("com.lihaoyi", "pprint_2.13"), -// ("com.lihaoyi", "upickle-core_2.13"), -// ("io.get-coursier.jniutils", "windows-jni-utils"), -// ("javax.inject", "javax.inject"), -// ("io.github.java-diff-utils", "java-diff-utils"), -// ("io.get-coursier", "interface"), -// ("org.scala-lang", "scala-compiler"), -// ("org.scala-lang", "scala-library"), -// ("io.get-coursier", "coursier-core_2.13"), -// ("org.iq80.snappy", "snappy"), -// ("org.fusesource.jansi", "jansi"), -// ("org.jline", "jline"), -// ("io.github.alexarchambault", "concurrent-reference-hash-map"), -// ("commons-io", "commons-io"), -// ("io.get-coursier", "coursier-util_2.13"), -// ("org.scala-sbt", "test-interface"), -// ("org.ow2.asm", "asm-tree"), -// ("org.ow2.asm", "asm-commons"), -// ("com.github.plokhotnyuk.jsoniter-scala", "jsoniter-scala-core_2.13"), -// ("org.scala-lang", "scala-reflect"), -// ("com.kohlschutter.junixsocket", "junixsocket-core"), -// ("com.lihaoyi", "geny_2.13"), -// ("com.typesafe", "config"), -// ("org.codehaus.plexus", "plexus-classworlds"), -// ("org.slf4j", "slf4j-api"), -// ("io.github.alexarchambault.windows-ansi", "windows-ansi"), -// ("org.ow2.asm", "asm"), -// ("com.lihaoyi", "mill-moduledefs_2.13") ) val tests: Tests = Tests { @@ -96,8 +34,6 @@ object MillPluginClasspathTest extends IntegrationTestSuite { assert(res1) val exclusions = metaValue[Seq[(String, String)]]("mill-build.resolveDepsExclusions") - - // pprint.pprintln(exclusions) val expectedExclusions = embeddedModules val diff = expectedExclusions.toSet.diff(exclusions.toSet) @@ -110,12 +46,10 @@ object MillPluginClasspathTest extends IntegrationTestSuite { assert(res1) val runClasspath = metaValue[Seq[String]]("mill-build.runClasspath") -// pprint.pprintln(runClasspath) val unexpectedArtifacts = embeddedModules.map { case (o, n) => s"${o.replaceAll("[.]", "/")}/${n}" } -// pprint.pprintln(unexpectedArtifacts) val unexpected = unexpectedArtifacts.flatMap { a => runClasspath.find(p => p.toString.contains(a)).map((a, _))