Overhaul transitive module handling in dependency resolution

bsp/worker/src/mill/bsp/worker/MillBuildServer.scala

@@ -296,7 +296,10 @@ private class MillBuildServer(
           Task.Anon {
             (
               m.defaultResolver().resolveDeps(
-                m.transitiveCompileIvyDeps() ++ m.transitiveIvyDeps(),
+                Seq(
+                  m.coursierDependency.withConfiguration(coursier.core.Configuration.provided),
+                  m.coursierDependency
+                ),
                 sources = true
               ),
               m.unmanagedClasspath(),
@@ -332,22 +335,33 @@ private class MillBuildServer(
       hint = "buildTargetDependencyModules",
       targetIds = _ => params.getTargets.asScala.toSeq,
       tasks = { case m: JavaModule =>
-        Task.Anon { (m.transitiveCompileIvyDeps(), m.transitiveIvyDeps(), m.unmanagedClasspath()) }
+        Task.Anon {
+          (
+            // full list of dependencies, including transitive ones
+            m.defaultResolver().allDeps(
+              Seq(
+                m.coursierDependency.withConfiguration(coursier.core.Configuration.provided),
+                m.coursierDependency
+              )
+            ),
+            m.unmanagedClasspath()
+          )
+        }
       }
     ) {
       case (
             ev,
             state,
             id,
             m: JavaModule,
-            (transitiveCompileIvyDeps, transitiveIvyDeps, unmanagedClasspath)
+            (ivyDeps, unmanagedClasspath)
           ) =>
-        val ivy = transitiveCompileIvyDeps ++ transitiveIvyDeps
-        val deps = ivy.map { dep =>
-          // TODO: add data with "maven" data kind using a ...
+        val deps = ivyDeps.collect {
+          case dep if dep.module.organization != JavaModule.internalOrg =>
+            // TODO: add data with "maven" data kind using a ...
 //          MavenDependencyModule
 
-          new DependencyModule(dep.dep.module.repr, dep.dep.version)
+            new DependencyModule(dep.module.repr, dep.version)
         }
         val unmanaged = unmanagedClasspath.map { dep =>
           new DependencyModule(s"unmanaged-${dep.path.last}", "")

build.mill

@@ -591,6 +591,10 @@ trait MillStableScalaModule extends MillPublishScalaModule with Mima {
       "mill.main.MainModule.mill$define$BaseModule0$_setter_$evalWatchedValues_="
     ),
 
+    // Overrides added for new methods, ought to be safe
+    ProblemFilter.exclude[ReversedMissingMethodProblem]("mill.scalalib.JavaModule.mill$scalalib$JavaModule$$super$internalRepositories"),
+    ProblemFilter.exclude[ReversedMissingMethodProblem]("mill.scalanativelib.ScalaNativeModule.mill$scalanativelib$ScalaNativeModule$$super$coursierProject"),
+
     // https://github.com/com-lihaoyi/mill/pull/3503
     ProblemFilter.exclude[ReversedMissingMethodProblem](
       "mill.scalalib.ScalaModule#ScalaTests.mill$scalalib$ScalaModule$ScalaTests$$super$mandatoryScalacOptions"

contrib/bloop/src/mill/contrib/bloop/BloopImpl.scala

@@ -392,8 +392,10 @@ class BloopImpl(evs: () => Seq[Evaluator], wd: os.Path) extends ExternalModule {
     val bloopResolution: Task[BloopConfig.Resolution] = Task.Anon {
       val repos = module.repositoriesTask()
       // same as input of resolvedIvyDeps
-      val allIvyDeps = module.transitiveIvyDeps() ++ module.transitiveCompileIvyDeps()
-      val coursierDeps = allIvyDeps.map(_.dep).toList
+      val coursierDeps = Seq(
+        module.coursierDependency.withConfiguration(coursier.core.Configuration.provided),
+        module.coursierDependency
+      )
       BloopConfig.Resolution(artifacts(repos, coursierDeps))
     }
 

contrib/playlib/src/mill/playlib/Static.scala

@@ -44,10 +44,12 @@ trait Static extends ScalaModule {
   }
 
   /**
-   * webjar dependencies - created from transitive ivy deps
+   * webjar dependencies - created from ivy deps
    */
   def webJarDeps = Task {
-    transitiveIvyDeps().filter(_.dep.module.organization.value == "org.webjars")
+    ivyDeps()
+      .filter(_.dep.module.organization.value == "org.webjars")
+      .map(bindDependency())
   }
 
   /**

contrib/scalapblib/src/mill/contrib/scalapblib/ScalaPBModule.scala

@@ -86,7 +86,12 @@ trait ScalaPBModule extends ScalaModule {
   else Task { Seq.empty[PathRef] }
 
   def scalaPBProtoClasspath: T[Agg[PathRef]] = Task {
-    defaultResolver().resolveDeps(transitiveCompileIvyDeps() ++ transitiveIvyDeps())
+    defaultResolver().resolveDeps(
+      Seq(
+        coursierDependency.withConfiguration(coursier.core.Configuration.provided),
+        coursierDependency
+      )
+    )
   }
 
   def scalaPBUnpackProto: T[PathRef] = Task {

contrib/scoverage/src/mill/contrib/scoverage/ScoverageModule.scala

@@ -158,7 +158,9 @@ trait ScoverageModule extends ScalaModule { outer: ScalaModule =>
     override def sources: T[Seq[PathRef]] = Task { outer.sources() }
     override def resources: T[Seq[PathRef]] = Task { outer.resources() }
     override def scalaVersion = Task { outer.scalaVersion() }
-    override def repositoriesTask: Task[Seq[Repository]] = Task.Anon { outer.repositoriesTask() }
+    override def repositoriesTask: Task[Seq[Repository]] = Task.Anon {
+      internalRepositories() ++ outer.repositoriesTask()
+    }
     override def compileIvyDeps: T[Agg[Dep]] = Task { outer.compileIvyDeps() }
     override def ivyDeps: T[Agg[Dep]] =
       Task { outer.ivyDeps() ++ outer.scoverageRuntimeDeps() }

example/cli/builtins/1-builtin-commands/build.mill

@@ -248,7 +248,6 @@ foo.unmanagedClasspath
 foo.compileIvyDeps
 ...
 foo.ivyDeps
-foo.transitiveIvyDeps
 foo.compileClasspath
 
 */

example/fundamentals/out-dir/1-out-files/build.mill

@@ -243,7 +243,8 @@ out/mill-server
 
 > cat out/mill-build/methodCodeHashSignatures.dest/current/spanningInvalidationTree.json
 {
-  "call scala.runtime.BoxesRunTime.boxToInteger(int)java.lang.Integer": {
+  "call scala.runtime.BoxesRunTime.boxToInteger(int)java.lang.Integer": {},
+  "call scala.Predef$#println(java.lang.Object)void": {
     "def build_.package_$foo$#<init>(build_.package_)void": {
       "call build_.package_$foo$!<init>(build_.package_)void": {
         "def build_.package_#foo$lzycompute$1()void": {
@@ -253,8 +254,7 @@ out/mill-server
         }
       }
     }
-  },
-  "call scala.Predef$#println(java.lang.Object)void": {}
+  }
 }
 
 */

idea/src/mill/idea/GenIdeaImpl.scala

@@ -132,7 +132,10 @@ case class GenIdeaImpl(
 
             // same as input of resolvedIvyDeps
             val allIvyDeps = Task.Anon {
-              mod.transitiveIvyDeps() ++ mod.transitiveCompileIvyDeps()
+              Agg(
+                mod.coursierDependency,
+                mod.coursierDependency.withConfiguration(coursier.core.Configuration.provided)
+              ).map(BoundDep(_, force = false))
             }
 
             val scalaCompilerClasspath = mod match {

integration/feature/docannotations/src/DocAnnotationsTests.scala

@@ -102,7 +102,12 @@ object DocAnnotationsTests extends UtestIntegrationTestSuite {
           |                             tree.
           |
           |Inputs:
-          |    core.transitiveIvyDeps
+          |    core.mandatoryIvyDeps
+          |    core.ivyDeps
+          |    core.compileIvyDeps
+          |    core.runIvyDeps
+          |    core.bomIvyDeps
+          |    core.depManagement
           |""".stripMargin,
         ivyDepsTree
       )

integration/ide/bsp-server/resources/snapshots/build-targets-dependency-modules.json

@@ -27,6 +27,10 @@
         {
           "name": "org.slf4j:slf4j-api",
           "version": "2.0.16"
+        },
+        {
+          "name": "org.hamcrest:hamcrest-core",
+          "version": "1.3"
         }
       ]
     },
@@ -44,6 +48,10 @@
         {
           "name": "org.jetbrains.kotlin:kotlin-stdlib",
           "version": "<kotlin-version>"
+        },
+        {
+          "name": "org.jetbrains:annotations",
+          "version": "13.0"
         }
       ]
     },

integration/ide/bsp-server/resources/snapshots/build-targets-jvm-run-environments.json

@@ -5,8 +5,8 @@
         "uri": "file:///workspace/app"
       },
       "classpath": [
-        "file:///coursier-cache/https/repo1.maven.org/maven2/com/mysql/mysql-connector-j/9.1.0/mysql-connector-j-9.1.0.jar",
         "file:///coursier-cache/https/repo1.maven.org/maven2/ch/qos/logback/logback-core/1.5.15/logback-core-1.5.15.jar",
+        "file:///coursier-cache/https/repo1.maven.org/maven2/com/mysql/mysql-connector-j/9.1.0/mysql-connector-j-9.1.0.jar",
         "file:///coursier-cache/https/repo1.maven.org/maven2/org/slf4j/slf4j-api/2.0.16/slf4j-api-2.0.16.jar",
         "file:///coursier-cache/https/repo1.maven.org/maven2/com/google/protobuf/protobuf-java/4.26.1/protobuf-java-4.26.1.jar",
         "file:///workspace/lib/compile-resources",

integration/ide/bsp-server/resources/snapshots/build-targets-jvm-test-environments.json

@@ -5,8 +5,8 @@
         "uri": "file:///workspace/app"
       },
       "classpath": [
-        "file:///coursier-cache/https/repo1.maven.org/maven2/com/mysql/mysql-connector-j/9.1.0/mysql-connector-j-9.1.0.jar",
         "file:///coursier-cache/https/repo1.maven.org/maven2/ch/qos/logback/logback-core/1.5.15/logback-core-1.5.15.jar",
+        "file:///coursier-cache/https/repo1.maven.org/maven2/com/mysql/mysql-connector-j/9.1.0/mysql-connector-j-9.1.0.jar",
         "file:///coursier-cache/https/repo1.maven.org/maven2/org/slf4j/slf4j-api/2.0.16/slf4j-api-2.0.16.jar",
         "file:///coursier-cache/https/repo1.maven.org/maven2/com/google/protobuf/protobuf-java/4.26.1/protobuf-java-4.26.1.jar",
         "file:///workspace/lib/compile-resources",

integration/ide/gen-idea/resources/hello-idea/idea/mill_modules/helloideajs.test.iml

@@ -11,7 +11,7 @@
         <orderEntry type="library" name="scala-js-SDK-3.3.1" level="project"/>
         <orderEntry type="library" name="portable-scala-reflect_sjs1_2.13-1.1.3.jar" level="project"/>
         <orderEntry type="library" name="scala-library-2.13.13.jar" level="project"/>
-        <orderEntry type="library" name="scala3-library_sjs1_3-3.3.1.jar" level="project"/>
+        <orderEntry type="library" name="scala3-library_sjs1_3-3.3.3.jar" level="project"/>
         <orderEntry type="library" name="scalajs-javalib-1.16.0.jar" level="project"/>
         <orderEntry type="library" name="scalajs-library_2.13-1.16.0.jar" level="project"/>
         <orderEntry type="library" name="scalajs-scalalib_2.13-2.13.13%2B1.16.0.jar" level="project"/>

main/package.mill

@@ -53,6 +53,8 @@ object `package` extends RootModule with build.MillStableScalaModule with BuildI
           //   Lib.resolveDependenciesMetadataSafe(
           Lib.resolveDependenciesMetadata(
             repositories = build.dist.repositoriesTask(),
+            // When re-bootstraping Mill, replace the next line with
+            //   Seq(BoundDep(build.dist.coursierDependency(), force = false)),
             build.dist.transitiveIvyDeps(),
             Some(build.dist.mapDependencies()),
             build.dist.resolutionCustomizer(),

main/util/src/mill/util/CoursierSupport.scala

@@ -1,17 +1,19 @@
 package mill.util
 
 import coursier.cache.{CacheLogger, FileCache}
-import coursier.core.BomDependency
+import coursier.core.{ArtifactSource, BomDependency, Extension, Info, Module, Project, Publication}
 import coursier.error.FetchError.DownloadingArtifacts
 import coursier.error.ResolutionError.CantDownloadModule
 import coursier.params.ResolutionParams
 import coursier.parse.RepositoryParser
 import coursier.jvm.{JvmCache, JvmChannel, JvmIndex, JavaHome}
-import coursier.util.Task
+import coursier.util.{Artifact, EitherT, Monad, Task}
 import coursier.{Artifacts, Classifier, Dependency, Repository, Resolution, Resolve, Type}
 import mill.api.Loose.Agg
 import mill.api.{Ctx, PathRef, Result}
 
+import java.util.concurrent.ConcurrentHashMap
+
 import scala.collection.mutable
 import scala.util.chaining.scalaUtilChainingOps
 import coursier.cache.ArchiveCache
@@ -32,18 +34,91 @@ trait CoursierSupport {
         ctx.fold(cache)(c => cache.withLogger(new TickerResolutionLogger(c)))
       }
 
-  private def isLocalTestDep(dep: Dependency): Option[Seq[PathRef]] = {
-    val org = dep.module.organization.value
-    val name = dep.module.name.value
-    val classpathKey = s"$org-$name"
+  /**
+   * A `coursier.Repository` that exposes modules with hard-coded artifact list
+   *
+   * Used in Mill tests. This exposes internal workers for example, so that these
+   * come from the build and not from remote repositories or ~/.ivy2/local. See
+   * `MillJavaModule#{testTransitiveDeps,writeLocalTestOverrides}` in the Mill build.
+   */
+  private final class TestOverridesRepo(root: os.ResourcePath) extends Repository {
+
+    private val map = new ConcurrentHashMap[Module, Option[String]]
+
+    private def listFor(mod: Module): Either[os.ResourceNotFoundException, String] = {
+
+      def entryPath = root / s"${mod.organization.value}-${mod.name.value}"
+
+      val inCacheOpt = Option(map.get(mod))
+
+      inCacheOpt
+        .getOrElse {
+
+          val computedOpt =
+            try Some(os.read(entryPath))
+            catch {
+              case _: os.ResourceNotFoundException =>
+                None
+            }
+          val concurrentOpt = Option(map.putIfAbsent(mod, computedOpt))
+          concurrentOpt.getOrElse(computedOpt)
+        }
+        .toRight {
+          new os.ResourceNotFoundException(entryPath)
+        }
+    }
 
-    val classpathResourceText =
-      try Some(os.read(
-          os.resource(getClass.getClassLoader) / "mill/local-test-overrides" / classpathKey
-        ))
-      catch { case e: os.ResourceNotFoundException => None }
+    def find[F[_]: Monad](
+        module: Module,
+        version: String,
+        fetch: Repository.Fetch[F]
+    ): EitherT[F, String, (ArtifactSource, Project)] =
+      EitherT.fromEither[F] {
+        listFor(module)
+          .left.map(e => s"No test override found at ${e.path}")
+          .map { _ =>
+            val proj = Project(
+              module,
+              version,
+              dependencies = Nil,
+              configurations = Map.empty,
+              parent = None,
+              dependencyManagement = Nil,
+              properties = Nil,
+              profiles = Nil,
+              versions = None,
+              snapshotVersioning = None,
+              packagingOpt = None,
+              relocated = false,
+              actualVersionOpt = None,
+              publications = Nil,
+              info = Info.empty
+            )
+            (this, proj)
+          }
+      }
 
-    classpathResourceText.map(_.linesIterator.map(s => PathRef(os.Path(s))).toSeq)
+    def artifacts(
+        dependency: Dependency,
+        project: Project,
+        overrideClassifiers: Option[Seq[Classifier]]
+    ): Seq[(Publication, Artifact)] =
+      listFor(project.module)
+        .toTry.get
+        .linesIterator
+        .map(os.Path(_))
+        .filter(os.exists)
+        .map { path =>
+          val pub = Publication(
+            if (path.last.endsWith(".jar")) path.last.stripSuffix(".jar") else path.last,
+            Type.jar,
+            Extension.jar,
+            Classifier.empty
+          )
+          val art = Artifact(path.toNIO.toUri.toASCIIString)
+          (pub, art)
+        }
+        .toSeq
   }
 
   /**
@@ -74,12 +149,9 @@ trait CoursierSupport {
       artifactTypes: Option[Set[Type]] = None,
       resolutionParams: ResolutionParams = ResolutionParams()
   ): Result[Agg[PathRef]] = {
-    val (localTestDeps, remoteDeps) =
-      deps.iterator.toSeq.partitionMap(d => isLocalTestDep(d).toLeft(d))
-
     val resolutionRes = resolveDependenciesMetadataSafe(
       repositories,
-      remoteDeps,
+      deps,
       force,
       mapDependencies,
       customizer,
@@ -118,7 +190,7 @@ trait CoursierSupport {
                 .map(os.Path(_))
                 .filter(deprecatedResolveFilter)
                 .map(PathRef(_, quick = true))
-            ) ++ localTestDeps.flatten
+            )
           )
       }
     }
@@ -275,7 +347,6 @@ trait CoursierSupport {
 
     val rootDeps = deps.iterator
       .map(d => mapDependencies.fold(d)(_.apply(d)))
-      .filter(dep => isLocalTestDep(dep).isEmpty)
       .toSeq
 
     val forceVersions = force.iterator
@@ -288,10 +359,13 @@ trait CoursierSupport {
     val resolutionParams0 = resolutionParams
       .addForceVersion(forceVersions.toSeq: _*)
 
+    val testOverridesRepo =
+      new TestOverridesRepo(os.resource(getClass.getClassLoader) / "mill/local-test-overrides")
+
     val resolve = Resolve()
       .withCache(coursierCache0)
       .withDependencies(rootDeps)
-      .withRepositories(repositories)
+      .withRepositories(testOverridesRepo +: repositories)
       .withResolutionParams(resolutionParams0)
       .withMapDependenciesOpt(mapDependencies)
       .withBoms(boms.toSeq)