diff --git a/docs/modules/ROOT/pages/fundamentals/library-deps.adoc b/docs/modules/ROOT/pages/fundamentals/library-deps.adoc index 8c703ab8175..b5e4c295e98 100644 --- a/docs/modules/ROOT/pages/fundamentals/library-deps.adoc +++ b/docs/modules/ROOT/pages/fundamentals/library-deps.adoc @@ -96,6 +96,34 @@ def runIvyDeps = Agg( It is also possible to use a higher version of the same library dependencies already defined in `ivyDeps`, to ensure you compile against a minimal API version, but actually run with the latest available version. +== Dependency management + +Dependency management consists in listing dependencies whose versions we want to force. Having +a dependency in dependency management doesn't mean that this dependency will be fetched, only +that + +* if it ends up being fetched transitively, its version will be forced to the one in dependency management + +* if its version is empty in an `ivyDeps` section in Mill, the version from dependency management will be used + +Dependency management also allows to add exclusions to dependencies, both explicit dependencies and +transitive ones. + +Dependency management can be passed to Mill in two ways: + +* via external Maven BOMs, like https://repo1.maven.org/maven2/com/google/cloud/libraries-bom/26.50.0/libraries-bom-26.50.0.pom[this one], +whose Maven coordinates are `com.google.cloud:libraries-bom:26.50.0` + +* via the `depManagement` task, that allows to directly list dependencies whose versions we want to enforce + +=== External BOMs + +include::partial$example/fundamentals/library-deps/bom-1-external-bom.adoc[] + +=== Dependency management task + +include::partial$example/fundamentals/library-deps/bom-2-dependency-management.adoc[] + == Searching For Dependency Updates include::partial$example/fundamentals/dependencies/1-search-updates.adoc[] diff --git a/docs/modules/ROOT/pages/javalib/dependencies.adoc b/docs/modules/ROOT/pages/javalib/dependencies.adoc index 76ebe8d4449..ac2c01aef22 100644 --- a/docs/modules/ROOT/pages/javalib/dependencies.adoc +++ b/docs/modules/ROOT/pages/javalib/dependencies.adoc @@ -13,6 +13,11 @@ include::partial$example/javalib/dependencies/1-ivy-deps.adoc[] include::partial$example/javalib/dependencies/2-run-compile-deps.adoc[] +== Dependency Management + +Mill has support for dependency management, see the +xref:fundamentals/library-deps.adoc#_dependency_management[Dependency Management section] +in xref:fundamentals/library-deps.adoc[]. == Unmanaged Jars diff --git a/docs/modules/ROOT/pages/kotlinlib/dependencies.adoc b/docs/modules/ROOT/pages/kotlinlib/dependencies.adoc index a970a256d4e..9670e311702 100644 --- a/docs/modules/ROOT/pages/kotlinlib/dependencies.adoc +++ b/docs/modules/ROOT/pages/kotlinlib/dependencies.adoc @@ -19,6 +19,12 @@ include::partial$example/kotlinlib/dependencies/1-ivy-deps.adoc[] include::partial$example/kotlinlib/dependencies/2-run-compile-deps.adoc[] +== Dependency Management + +Mill has support for dependency management, see the +xref:fundamentals/library-deps.adoc#_dependency_management[Dependency Management section] +in xref:fundamentals/library-deps.adoc[]. + == Unmanaged Jars include::partial$example/kotlinlib/dependencies/3-unmanaged-jars.adoc[] diff --git a/docs/modules/ROOT/pages/scalalib/dependencies.adoc b/docs/modules/ROOT/pages/scalalib/dependencies.adoc index c4c0bf2ef89..902d8b7be15 100644 --- a/docs/modules/ROOT/pages/scalalib/dependencies.adoc +++ b/docs/modules/ROOT/pages/scalalib/dependencies.adoc @@ -16,6 +16,11 @@ include::partial$example/scalalib/dependencies/1-ivy-deps.adoc[] include::partial$example/scalalib/dependencies/2-run-compile-deps.adoc[] +== Dependency Management + +Mill has support for dependency management, see the +xref:fundamentals/library-deps.adoc#_dependency_management[Dependency Management section] +in xref:fundamentals/library-deps.adoc[]. == Unmanaged Jars diff --git a/example/fundamentals/library-deps/bom-1-external-bom/build.mill b/example/fundamentals/library-deps/bom-1-external-bom/build.mill new file mode 100644 index 00000000000..4b076da2999 --- /dev/null +++ b/example/fundamentals/library-deps/bom-1-external-bom/build.mill @@ -0,0 +1,25 @@ +// Pass an external BOM to a `JavaModule` / `ScalaModule` / `KotlinModule` with `bomDeps`, like + +//// SNIPPET:BUILD1 +package build +import mill._, javalib._ + +object foo extends JavaModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0" + ) + def ivyDeps = Agg( + ivy"io.grpc:grpc-protobuf" + ) +} + +// The version of grpc-protobuf (`io.grpc:grpc-protobuf`) isn't written down here, so the version +// from the BOM, `1.67.1` is used. +// +// Also, by default, grpc-protobuf `1.67.1` pulls version `3.25.3` of protobuf-java (`com.google.protobuf:protobuf-java`) . +// But the BOM specifies another version for that dependency, `4.28.3`, so +// protobuf-java `4.28.3` ends up being pulled here. +// +// Several BOMs can be passed to `bomDeps`. If several specify a version for a dependency, +// the version from the first one in the `bomDeps` list is used. If several specify exclusions +// for a dependency, all exclusions are added to that dependency. diff --git a/example/fundamentals/library-deps/bom-2-dependency-management/build.mill b/example/fundamentals/library-deps/bom-2-dependency-management/build.mill new file mode 100644 index 00000000000..6caf0b6b99d --- /dev/null +++ b/example/fundamentals/library-deps/bom-2-dependency-management/build.mill @@ -0,0 +1,54 @@ +// Pass dependencies to `depManagement` in a `JavaModule` / `ScalaModule` / `KotlinModule`, like + +//// SNIPPET:BUILD1 +package build +import mill._, javalib._ + +object foo extends JavaModule { + def depManagement = Agg( + ivy"com.google.protobuf:protobuf-java:4.28.3", + ivy"io.grpc:grpc-protobuf:1.67.1" + ) + def ivyDeps = Agg( + ivy"io.grpc:grpc-protobuf" + ) +} + +// The version of grpc-protobuf (`io.grpc:grpc-protobuf`) isn't written down here, so the version +// found in `depManagement`, `1.67.1` is used. +// +// Also, by default, grpc-protobuf `1.67.1` pulls version `3.25.3` of protobuf-java (`com.google.protobuf:protobuf-java`) . +// But `depManagement` specifies another version for that dependency, `4.28.3`, so +// protobuf-java `4.28.3` ends up being pulled here. + +// One can also add exclusions via dependency management, like + +object bar extends JavaModule { + def depManagement = Agg( + ivy"io.grpc:grpc-protobuf:1.67.1" + .exclude(("com.google.protobuf", "protobuf-java")) + ) + def ivyDeps = Agg( + ivy"io.grpc:grpc-protobuf" + ) +} + +// Here, grpc-protobuf has an empty version in `ivyDeps`, so the one in `depManagement`, +// `1.67.1`, is used. Also, `com.google.protobuf:protobuf-java` is excluded from grpc-protobuf +// in `depManagement`, so it ends up being excluded from it in `ivyDeps` too. + +// If one wants to add exclusions via `depManagement`, specifying a version is optional, +// like + +object baz extends JavaModule { + def depManagement = Agg( + ivy"io.grpc:grpc-protobuf" + .exclude(("com.google.protobuf", "protobuf-java")) + ) + def ivyDeps = Agg( + ivy"io.grpc:grpc-protobuf:1.67.1" + ) +} + +// Here, given that grpc-protobuf is fetched during dependency resolution, +// `com.google.protobuf:protobuf-java` is excluded from it because of the dependency management. diff --git a/example/package.mill b/example/package.mill index f65a573596e..1f214fe09b2 100644 --- a/example/package.mill +++ b/example/package.mill @@ -82,6 +82,7 @@ object `package` extends RootModule with Module { object cross extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "cross")) object `out-dir` extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "out-dir")) object libraries extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "libraries")) + object `library-deps` extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "library-deps")) } object depth extends Module { diff --git a/main/util/src/mill/util/CoursierSupport.scala b/main/util/src/mill/util/CoursierSupport.scala index c44b32c3c65..6f6ccff62b2 100644 --- a/main/util/src/mill/util/CoursierSupport.scala +++ b/main/util/src/mill/util/CoursierSupport.scala @@ -31,6 +31,20 @@ trait CoursierSupport { ctx.fold(cache)(c => cache.withLogger(new TickerResolutionLogger(c))) } + private def isLocalTestDep(dep: Dependency): Option[Seq[PathRef]] = { + val org = dep.module.organization.value + val name = dep.module.name.value + val classpathKey = s"$org-$name" + + val classpathResourceText = + try Some(os.read( + os.resource(getClass.getClassLoader) / "mill/local-test-overrides" / classpathKey + )) + catch { case e: os.ResourceNotFoundException => None } + + classpathResourceText.map(_.linesIterator.map(s => PathRef(os.Path(s))).toSeq) + } + /** * Resolve dependencies using Coursier. * @@ -51,26 +65,8 @@ trait CoursierSupport { artifactTypes: Option[Set[Type]] = None, resolutionParams: ResolutionParams = ResolutionParams() ): Result[Agg[PathRef]] = { - def isLocalTestDep(dep: Dependency): Option[Seq[PathRef]] = { - val org = dep.module.organization.value - val name = dep.module.name.value - val classpathKey = s"$org-$name" - - val classpathResourceText = - try Some(os.read( - os.resource(getClass.getClassLoader) / "mill/local-test-overrides" / classpathKey - )) - catch { case e: os.ResourceNotFoundException => None } - - classpathResourceText.map(_.linesIterator.map(s => PathRef(os.Path(s))).toSeq) - } - - val (localTestDeps, remoteDeps) = deps.iterator.toSeq.partitionMap(d => - isLocalTestDep(d) match { - case None => Right(d) - case Some(vs) => Left(vs) - } - ) + val (localTestDeps, remoteDeps) = + deps.iterator.toSeq.partitionMap(d => isLocalTestDep(d).toLeft(d)) val resolutionRes = resolveDependenciesMetadataSafe( repositories, @@ -262,6 +258,7 @@ trait CoursierSupport { val rootDeps = deps.iterator .map(d => mapDependencies.fold(d)(_.apply(d))) + .filter(dep => isLocalTestDep(dep).isEmpty) .toSeq val forceVersions = force.iterator diff --git a/scalalib/src/mill/scalalib/CoursierModule.scala b/scalalib/src/mill/scalalib/CoursierModule.scala index 0b687f66646..c7c483cceb1 100644 --- a/scalalib/src/mill/scalalib/CoursierModule.scala +++ b/scalalib/src/mill/scalalib/CoursierModule.scala @@ -231,6 +231,37 @@ object CoursierModule { sources: Boolean ): Agg[PathRef] = resolveDeps(deps, sources, None) + + /** + * Processes dependencies and BOMs with coursier + * + * This makes coursier read and process BOM dependencies, and fill version placeholders + * in dependencies with the BOMs. + * + * Note that this doesn't throw when a version placeholder cannot be filled, and just leaves + * the placeholder behind. + * + * @param deps dependencies that might have placeholder versions ("_" as version) + * @param resolutionParams coursier resolution parameters + * @return dependencies with version placeholder filled + */ + def processDeps[T: CoursierModule.Resolvable]( + deps: IterableOnce[T], + resolutionParams: ResolutionParams = ResolutionParams() + ): Seq[Dependency] = { + val deps0 = deps + .map(implicitly[CoursierModule.Resolvable[T]].bind(_, bind)) + val res = Lib.resolveDependenciesMetadataSafe( + repositories = repositories, + deps = deps0, + mapDependencies = mapDependencies, + customizer = customizer, + coursierCacheCustomizer = coursierCacheCustomizer, + ctx = ctx, + resolutionParams = resolutionParams + ).getOrThrow + res.processedRootDependencies + } } sealed trait Resolvable[T] { diff --git a/scalalib/src/mill/scalalib/Dep.scala b/scalalib/src/mill/scalalib/Dep.scala index ac2569ada99..3b02d8daa4d 100644 --- a/scalalib/src/mill/scalalib/Dep.scala +++ b/scalalib/src/mill/scalalib/Dep.scala @@ -118,6 +118,8 @@ object Dep { } (module.split(':') match { + case Array(a, b) => Dep(a, b, "_", cross = empty(platformed = false)) + case Array(a, "", b) => Dep(a, b, "_", cross = Binary(platformed = false)) case Array(a, b, c) => Dep(a, b, c, cross = empty(platformed = false)) case Array(a, b, "", c) => Dep(a, b, c, cross = empty(platformed = true)) case Array(a, "", b, c) => Dep(a, b, c, cross = Binary(platformed = false)) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 1c901525557..3fe981d8508 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -1,7 +1,7 @@ package mill package scalalib -import coursier.core.Resolution +import coursier.core.{BomDependency, Configuration, DependencyManagement, Resolution} import coursier.parse.JavaOrScalaModule import coursier.parse.ModuleParser import coursier.util.ModuleMatcher @@ -159,6 +159,139 @@ trait JavaModule */ def runIvyDeps: T[Agg[Dep]] = Task { Agg.empty[Dep] } + /** + * Any BOM dependencies you want to add to this Module, in the format + * ivy"org:name:version" + */ + def bomDeps: T[Agg[Dep]] = Task { Agg.empty[Dep] } + + def allBomDeps: Task[Agg[BomDependency]] = Task.Anon { + val modVerOrMalformed = + bomDeps().map(bindDependency()).map { bomDep => + val fromModVer = coursier.core.Dependency(bomDep.dep.module, bomDep.dep.version) + if (fromModVer == bomDep.dep) + Right(bomDep.dep.asBomDependency) + else + Left(bomDep) + } + + val malformed = modVerOrMalformed.collect { + case Left(malformedBomDep) => + malformedBomDep + } + if (malformed.isEmpty) + modVerOrMalformed.collect { + case Right(bomDep) => bomDep + } + else + throw new Exception( + "Found BOM dependencies with invalid parameters:" + System.lineSeparator() + + malformed.map("- " + _.dep + System.lineSeparator()).mkString + + "Only organization, name, and version are accepted." + ) + } + + /** + * Dependency management data + * + * Versions and exclusions in dependency management override those of transitive dependencies, + * while they have no effect if the corresponding dependency isn't pulled during dependency + * resolution. + * + * For example, the following forces com.lihaoyi::os-lib to version 0.11.3, and + * excludes org.slf4j:slf4j-api from com.lihaoyi::cask that it forces to version 0.9.4 + * {{{ + * def depManagement = super.depManagement() ++ Agg( + * ivy"com.lihaoyi::os-lib:0.11.3", + * ivy"com.lihaoyi::cask:0.9.4".exclude("org.slf4j", "slf4j-api") + * ) + * }}} + */ + def depManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } + + private def addBoms( + bomDeps: Seq[coursier.core.BomDependency], + depMgmt: Seq[(DependencyManagement.Key, DependencyManagement.Values)], + overrideVersions: Boolean + ): coursier.core.Dependency => coursier.core.Dependency = { + val depMgmtMap = depMgmt.toMap + dep => + val depMgmtKey = DependencyManagement.Key( + dep.module.organization, + dep.module.name, + coursier.core.Type.jar, + dep.publication.classifier + ) + val versionOverrideOpt = + if (dep.version == "_") depMgmtMap.get(depMgmtKey).map(_.version) + else None + val extraExclusions = depMgmtMap.get(depMgmtKey).map(_.minimizedExclusions) + dep + // add BOM coordinates - coursier will handle the rest + .addBomDependencies( + if (overrideVersions) bomDeps.map(_.withForceOverrideVersions(overrideVersions)) + else bomDeps + ) + // add dependency management ourselves: + // - overrides meant to apply to transitive dependencies + // - fill version if it's empty + // - add extra exclusions from dependency management + .withOverrides(dep.overrides ++ depMgmt) + .withVersion(versionOverrideOpt.getOrElse(dep.version)) + .withMinimizedExclusions( + extraExclusions.fold(dep.minimizedExclusions)(dep.minimizedExclusions.join(_)) + ) + } + + /** + * Data from depManagement, converted to a type ready to be passed to coursier + * for dependency resolution + */ + private def processedDependencyManagement(deps: Seq[coursier.core.Dependency]) + : Seq[(DependencyManagement.Key, DependencyManagement.Values)] = { + val keyValuesOrErrors = + deps.map { depMgmt => + val fromUsedValues = coursier.core.Dependency(depMgmt.module, depMgmt.version) + .withPublication(coursier.core.Publication( + "", + depMgmt.publication.`type`, + coursier.core.Extension.empty, + depMgmt.publication.classifier + )) + .withMinimizedExclusions(depMgmt.minimizedExclusions) + .withOptional(depMgmt.optional) + if (fromUsedValues == depMgmt) { + val key = DependencyManagement.Key( + depMgmt.module.organization, + depMgmt.module.name, + if (depMgmt.publication.`type`.isEmpty) coursier.core.Type.jar + else depMgmt.publication.`type`, + depMgmt.publication.classifier + ) + val values = DependencyManagement.Values( + Configuration.empty, + if (depMgmt.version == "_") "" // shouldn't be needed with future coursier versions + else depMgmt.version, + depMgmt.minimizedExclusions, + depMgmt.optional + ) + Right(key -> values) + } else + Left(depMgmt) + } + + val errors = keyValuesOrErrors.collect { + case Left(errored) => errored + } + if (errors.isEmpty) + keyValuesOrErrors.collect { case Right(kv) => kv } + else + throw new Exception( + "Found dependency management entries with invalid values. Only organization, name, version, type, classifier, exclusions, and optionality can be specified" + System.lineSeparator() + + errors.map("- " + _ + System.lineSeparator()).mkString + ) + } + /** * Default artifact types to fetch and put in the classpath. Add extra types * here if you'd like fancy artifact extensions to be fetched. @@ -326,13 +459,43 @@ trait JavaModule */ def unmanagedClasspath: T[Agg[PathRef]] = Task { Agg.empty[PathRef] } + /** + * Returns a function adding BOM and dependency management details of + * this module to a `coursier.core.Dependency` + */ + def processDependency( + overrideVersions: Boolean = false + ): Task[coursier.core.Dependency => coursier.core.Dependency] = Task.Anon { + val bomDeps0 = allBomDeps().toSeq.map(_.withConfig(Configuration.compile)) + val depMgmt = processedDependencyManagement( + depManagement().toSeq.map(bindDependency()).map(_.dep) + ) + + addBoms(bomDeps0, depMgmt, overrideVersions = overrideVersions) + } + + /** + * The Ivy dependencies of this module, with BOM and dependency management details + * added to them. This should be used when propagating the dependencies transitively + * to other modules. + */ + def processedIvyDeps: Task[Agg[BoundDep]] = Task.Anon { + val processDependency0 = processDependency()() + allIvyDeps().map(bindDependency()).map { dep => + dep.copy(dep = processDependency0(dep.dep)) + } + } + /** * The transitive ivy dependencies of this module and all it's upstream modules. * This is calculated from [[ivyDeps]], [[mandatoryIvyDeps]] and recursively from [[moduleDeps]]. */ def transitiveIvyDeps: T[Agg[BoundDep]] = Task { - allIvyDeps().map(bindDependency()) ++ - T.traverse(moduleDepsChecked)(_.transitiveIvyDeps)().flatten + val processDependency0 = processDependency(overrideVersions = true)() + processedIvyDeps() ++ + T.traverse(moduleDepsChecked)(_.transitiveIvyDeps)().flatten.map { dep => + dep.copy(dep = processDependency0(dep.dep)) + } } /** diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index 796c3ff10f9..cc9fc686998 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -70,7 +70,7 @@ trait PublishModule extends JavaModule { outer => def publishXmlDeps: Task[Agg[Dependency]] = Task.Anon { val ivyPomDeps = - (ivyDeps() ++ mandatoryIvyDeps()).map(resolvePublishDependency.apply().apply(_)) + processedIvyDeps().map(_.toDep).map(resolvePublishDependency.apply().apply(_)) val compileIvyPomDeps = compileIvyDeps() .map(resolvePublishDependency.apply().apply(_)) @@ -89,6 +89,20 @@ trait PublishModule extends JavaModule { outer => compileModulePomDeps.map(Dependency(_, Scope.Provided)) } + /** + * BOM dependency to specify in the POM + */ + def publishXmlBomDeps: Task[Agg[Dependency]] = Task.Anon { + bomDeps().map(resolvePublishDependency.apply().apply(_)) + } + + /** + * Dependency management to specify in the POM + */ + def publishXmlDepMgmt: Task[Agg[Dependency]] = Task.Anon { + depManagement().map(resolvePublishDependency.apply().apply(_)) + } + def pom: T[PathRef] = Task { val pom = Pom( artifactMetadata(), @@ -97,15 +111,72 @@ trait PublishModule extends JavaModule { outer => pomSettings(), publishProperties(), packagingType = pomPackagingType, - parentProject = pomParentProject() + parentProject = pomParentProject(), + bomDependencies = publishXmlBomDeps(), + dependencyManagement = publishXmlDepMgmt() ) val pomPath = T.dest / s"${artifactId()}-${publishVersion()}.pom" os.write.over(pomPath, pom) PathRef(pomPath) } + /** + * Dependencies with version placeholder filled from BOMs, alongside with BOM data + */ + def bomDetails: T[(Map[coursier.core.Module, String], coursier.core.DependencyManagement.Map)] = + Task { + val processedDeps = defaultResolver().processDeps( + transitiveCompileIvyDeps() ++ transitiveIvyDeps(), + resolutionParams = resolutionParams() + ) + val depMgmt: coursier.core.DependencyManagement.Map = + if (processedDeps.isEmpty) Map.empty + else { + val overrides = processedDeps.map(_.overrides) + overrides.tail.foldLeft(overrides.head) { (acc, map) => + acc.filter { + case (key, values) => + map.get(key).contains(values) + } + } + } + (processedDeps.map(_.moduleVersion).toMap, depMgmt) + } + def ivy: T[PathRef] = Task { - val ivy = Ivy(artifactMetadata(), publishXmlDeps(), extraPublish()) + val (rootDepVersions, bomDepMgmt) = bomDetails() + val publishXmlDeps0 = publishXmlDeps().map { dep => + if (dep.artifact.version == "_") + dep.copy( + artifact = dep.artifact.copy( + version = rootDepVersions.getOrElse( + coursier.core.Module( + coursier.core.Organization(dep.artifact.group), + coursier.core.ModuleName(dep.artifact.id), + Map.empty + ), + "" /* throw instead? */ + ) + ) + ) + else + dep + } + val overrides = + depManagement().toSeq.map(bindDependency()).map(_.dep) + .filter(depMgmt => depMgmt.version.nonEmpty && depMgmt.version != "_") + .map { depMgmt => + Ivy.Override( + depMgmt.module.organization.value, + depMgmt.module.name.value, + depMgmt.version + ) + } ++ + bomDepMgmt.map { + case (key, values) => + Ivy.Override(key.organization.value, key.name.value, values.version) + } + val ivy = Ivy(artifactMetadata(), publishXmlDeps0, extraPublish(), overrides) val ivyPath = T.dest / "ivy.xml" os.write.over(ivyPath, ivy) PathRef(ivyPath) diff --git a/scalalib/src/mill/scalalib/publish/Ivy.scala b/scalalib/src/mill/scalalib/publish/Ivy.scala index ae340ac0d75..75ba33dbec1 100644 --- a/scalalib/src/mill/scalalib/publish/Ivy.scala +++ b/scalalib/src/mill/scalalib/publish/Ivy.scala @@ -8,10 +8,13 @@ object Ivy { val head = "\n" + case class Override(organization: String, name: String, version: String) + def apply( artifact: Artifact, dependencies: Agg[Dependency], - extras: Seq[PublishInfo] = Seq.empty + extras: Seq[PublishInfo] = Seq.empty, + overrides: Seq[Override] = Nil ): String = { def renderExtra(e: PublishInfo): Elem = { @@ -49,13 +52,29 @@ object Ivy { {extras.map(renderExtra)} - {dependencies.map(renderDependency).toSeq} + + {dependencies.map(renderDependency).toSeq} + {overrides.map(renderOverride)} + val pp = new PrettyPrinter(120, 4) head + pp.format(xml).replaceAll(">", ">") } + // bin-compat shim + def apply( + artifact: Artifact, + dependencies: Agg[Dependency], + extras: Seq[PublishInfo] + ): String = + apply( + artifact, + dependencies, + extras, + Nil + ) + private def renderDependency(dep: Dependency): Elem = { if (dep.exclusions.isEmpty) } + private def renderOverride(override0: Override): Elem = + + private def depIvyConf(d: Dependency): String = { if (d.optional) "optional" else d.scope match { diff --git a/scalalib/src/mill/scalalib/publish/Pom.scala b/scalalib/src/mill/scalalib/publish/Pom.scala index 76d1e728d36..6940f25c47f 100644 --- a/scalalib/src/mill/scalalib/publish/Pom.scala +++ b/scalalib/src/mill/scalalib/publish/Pom.scala @@ -39,10 +39,15 @@ object Pom { pomSettings = pomSettings, properties = properties, packagingType = pomSettings.packaging, - parentProject = None + parentProject = None, + bomDependencies = Agg.empty[Dependency], + dependencyManagement = Agg.empty[Dependency] ) - @deprecated("Use overload with parentProject parameter instead", "Mill 0.12.1") + @deprecated( + "Use overload with parentProject, bomDependencies, and dependencyManagement parameters instead", + "Mill 0.12.1" + ) def apply( artifact: Artifact, dependencies: Agg[Dependency], @@ -57,7 +62,9 @@ object Pom { pomSettings = pomSettings, properties = properties, packagingType = packagingType, - parentProject = None + parentProject = None, + bomDependencies = Agg.empty[Dependency], + dependencyManagement = Agg.empty[Dependency] ) def apply( @@ -68,6 +75,29 @@ object Pom { properties: Map[String, String], packagingType: String, parentProject: Option[Artifact] + ): String = + apply( + artifact, + dependencies, + name, + pomSettings, + properties, + packagingType, + parentProject, + Agg.empty[Dependency], + Agg.empty[Dependency] + ) + + def apply( + artifact: Artifact, + dependencies: Agg[Dependency], + name: String, + pomSettings: PomSettings, + properties: Map[String, String], + packagingType: String, + parentProject: Option[Artifact], + bomDependencies: Agg[Dependency], + dependencyManagement: Agg[Dependency] ): String = { val xml = - {dependencies.map(renderDependency).iterator} + { + dependencies.map(renderDependency(_)).iterator ++ + bomDependencies.map(renderDependency(_, isImport = true)).iterator + } + + + {dependencyManagement.map(renderDependency(_)).iterator} + + val pp = new PrettyPrinter(120, 4) @@ -143,29 +181,39 @@ object Pom { {property._2}.copy(label = property._1) } - private def renderDependency(d: Dependency): Elem = { - val scope = d.scope match { - case Scope.Compile => NodeSeq.Empty - case Scope.Provided => provided - case Scope.Test => test - case Scope.Runtime => runtime - } + private def renderDependency(d: Dependency, isImport: Boolean = false): Elem = { + val scope = + if (isImport) import + else + d.scope match { + case Scope.Compile => NodeSeq.Empty + case Scope.Provided => provided + case Scope.Test => test + case Scope.Runtime => runtime + } + + val `type` = if (isImport) pom else NodeSeq.Empty val optional = if (d.optional) true else NodeSeq.Empty + val version = + if (d.artifact.version == "_") NodeSeq.Empty + else {d.artifact.version} + if (d.exclusions.isEmpty) {d.artifact.group} {d.artifact.id} - {d.artifact.version} + {version} {scope} + {`type`} {optional} else {d.artifact.group} {d.artifact.id} - {d.artifact.version} + {version} { d.exclusions.map(ex => @@ -175,6 +223,7 @@ object Pom { } {scope} + {`type`} {optional} } diff --git a/scalalib/test/src/mill/scalalib/BomTests.scala b/scalalib/test/src/mill/scalalib/BomTests.scala new file mode 100644 index 00000000000..48fc81c9628 --- /dev/null +++ b/scalalib/test/src/mill/scalalib/BomTests.scala @@ -0,0 +1,484 @@ +package mill +package scalalib + +import mill.scalalib.publish._ +import mill.testkit.{TestBaseModule, UnitTester} +import utest._ + +import scala.jdk.CollectionConverters._ + +object BomTests extends TestSuite { + + trait TestPublishModule extends PublishModule { + def pomSettings = PomSettings( + description = artifactName(), + organization = "com.lihaoyi.mill-tests", + url = "https://github.com/com-lihaoyi/mill", + licenses = Seq(License.`Apache-2.0`), + versionControl = VersionControl.github("com-lihaoyi", "mill"), + developers = Nil + ) + def publishVersion = "0.1.0-SNAPSHOT" + } + + object modules extends TestBaseModule { + object bom extends Module { + object placeholder extends JavaModule with TestPublishModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0" + ) + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java" + ) + + object dependee extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + placeholder + ) + } + + object subDependee extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + dependee + ) + } + + object check extends JavaModule { + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java" + ) + } + } + + object versionOverride extends JavaModule with TestPublishModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0" + ) + def ivyDeps = Agg( + ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" + ) + + object dependee extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + versionOverride + ) + } + + object subDependee extends JavaModule with TestPublishModule { + def moduleDeps = Seq( + dependee + ) + } + + object check extends JavaModule { + def ivyDeps = Agg( + ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" + ) + } + } + + object invalid extends TestBaseModule { + object exclude extends JavaModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0".exclude(("foo", "thing")) + ) + } + } + } + + object depMgmt extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.thesamet.scalapb:scalapbc_2.13:0.9.8" + ) + def depManagement = Agg( + ivy"com.google.protobuf:protobuf-java:4.28.3" + ) + + object transitive extends JavaModule with TestPublishModule { + def moduleDeps = Seq(depMgmt) + } + + object extraExclude extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.lihaoyi:cask_2.13:0.9.4" + ) + def depManagement = Agg( + // The exclude should be automatically added to the dependency above + // thanks to dependency management, but the version should be left + // untouched + ivy"com.lihaoyi:cask_2.13:0.9.3" + .exclude(("org.slf4j", "slf4j-api")) + ) + + object transitive extends JavaModule with TestPublishModule { + def moduleDeps = Seq(extraExclude) + } + } + + object exclude extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.lihaoyi:cask_2.13:0.9.4" + ) + def depManagement = Agg( + ivy"org.java-websocket:Java-WebSocket:1.5.2" + .exclude(("org.slf4j", "slf4j-api")) + ) + + object transitive extends JavaModule with TestPublishModule { + def moduleDeps = Seq(exclude) + } + } + + object onlyExclude extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.lihaoyi:cask_2.13:0.9.4" + ) + def depManagement = Agg( + ivy"org.java-websocket:Java-WebSocket" + .exclude(("org.slf4j", "slf4j-api")) + ) + + object transitive extends JavaModule with TestPublishModule { + def moduleDeps = Seq(onlyExclude) + } + } + + object invalid extends TestBaseModule { + object transitive extends JavaModule { + def depManagement = { + val dep = ivy"org.java-websocket:Java-WebSocket:1.5.3" + Agg( + dep.copy( + dep = dep.dep.withTransitive(false) + ) + ) + } + } + } + + object placeholder extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java" + ) + def depManagement = Agg( + ivy"com.google.protobuf:protobuf-java:4.28.3" + ) + + object transitive extends JavaModule with TestPublishModule { + def moduleDeps = Seq(placeholder) + } + } + } + + object bomOnModuleDependency extends JavaModule with TestPublishModule { + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java:3.23.4" + ) + + object dependee extends JavaModule with TestPublishModule { + def bomDeps = Agg( + ivy"com.google.cloud:libraries-bom:26.50.0" + ) + def moduleDeps = Seq(bomOnModuleDependency) + } + } + } + + def expectedProtobufJavaVersion = "4.28.3" + def expectedCommonsCompressVersion = "1.23.0" + + def expectedProtobufJarName = s"protobuf-java-$expectedProtobufJavaVersion.jar" + def expectedCommonsCompressJarName = s"commons-compress-$expectedCommonsCompressVersion.jar" + + def compileClasspathFileNames(module: JavaModule)(implicit + eval: UnitTester + ): Seq[String] = + eval(module.compileClasspath).toTry.get.value + .toSeq.map(_.path.last) + + def compileClasspathContains( + module: JavaModule, + fileName: String, + jarCheck: Option[String => Boolean] + )(implicit + eval: UnitTester + ) = { + val fileNames = compileClasspathFileNames(module) + assert(fileNames.contains(fileName)) + for (check <- jarCheck; fileName <- fileNames) + assert(check(fileName)) + } + + def publishLocalAndResolve( + module: PublishModule, + dependencyModules: Seq[PublishModule], + scalaSuffix: String + )(implicit eval: UnitTester): Seq[os.Path] = { + val localIvyRepo = eval.evaluator.workspace / "ivy2Local" + eval(module.publishLocal(localIvyRepo.toString)).toTry.get + for (dependencyModule <- dependencyModules) + eval(dependencyModule.publishLocal(localIvyRepo.toString)).toTry.get + + val moduleString = eval(module.artifactName).toTry.get.value + + coursierapi.Fetch.create() + .addDependencies( + coursierapi.Dependency.of( + "com.lihaoyi.mill-tests", + moduleString.replace('.', '-') + scalaSuffix, + "0.1.0-SNAPSHOT" + ) + ) + .addRepositories( + coursierapi.IvyRepository.of(localIvyRepo.toNIO.toUri.toASCIIString + "[defaultPattern]") + ) + .fetch() + .asScala + .map(os.Path(_)) + .toVector + } + + def publishM2LocalAndResolve( + module: PublishModule, + dependencyModules: Seq[PublishModule], + scalaSuffix: String + )(implicit eval: UnitTester): Seq[os.Path] = { + val localM2Repo = eval.evaluator.workspace / "m2Local" + eval(module.publishM2Local(localM2Repo.toString)).toTry.get + for (dependencyModule <- dependencyModules) + eval(dependencyModule.publishM2Local(localM2Repo.toString)).toTry.get + + val moduleString = eval(module.artifactName).toTry.get.value + + coursierapi.Fetch.create() + .addDependencies( + coursierapi.Dependency.of( + "com.lihaoyi.mill-tests", + moduleString.replace('.', '-') + scalaSuffix, + "0.1.0-SNAPSHOT" + ) + ) + .addRepositories( + coursierapi.MavenRepository.of(localM2Repo.toNIO.toUri.toASCIIString) + ) + .fetch() + .asScala + .map(os.Path(_)) + .toVector + } + + def isInClassPath( + module: JavaModule with PublishModule, + jarName: String, + dependencyModules: Seq[PublishModule] = Nil, + jarCheck: Option[String => Boolean] = None, + ivy2LocalCheck: Boolean = true, + scalaSuffix: String = "" + )(implicit eval: UnitTester): Unit = { + compileClasspathContains(module, jarName, jarCheck) + + if (ivy2LocalCheck) { + val resolvedCp = publishLocalAndResolve(module, dependencyModules, scalaSuffix) + assert(resolvedCp.map(_.last).contains(jarName)) + for (check <- jarCheck; fileName <- resolvedCp.map(_.last)) + assert(check(fileName)) + } + + val resolvedM2Cp = publishM2LocalAndResolve(module, dependencyModules, scalaSuffix) + assert(resolvedM2Cp.map(_.last).contains(jarName)) + for (check <- jarCheck; fileName <- resolvedM2Cp.map(_.last)) + assert(check(fileName)) + } + + def tests = Tests { + + test("bom") { + test("placeholder") { + test("check") - UnitTester(modules, null).scoped { eval => + val res = eval(modules.bom.placeholder.check.compileClasspath) + assert( + res.left.exists(_.toString.contains( + "not found: https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/_/protobuf-java-_.pom" + )) + ) + } + + test("simple") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath(modules.bom.placeholder, expectedProtobufJarName) + } + + test("dependee") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bom.placeholder.dependee, + expectedProtobufJarName, + Seq(modules.bom.placeholder) + ) + } + + test("subDependee") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bom.placeholder.subDependee, + expectedProtobufJarName, + Seq(modules.bom.placeholder, modules.bom.placeholder.dependee) + ) + } + } + + test("versionOverride") { + test("check") - UnitTester(modules, null).scoped { implicit eval => + val fileNames = compileClasspathFileNames(modules.bom.versionOverride.check) + assert(fileNames.exists(v => v.startsWith("protobuf-java-") && v.endsWith(".jar"))) + assert(!fileNames.contains(expectedProtobufJarName)) + } + + test("simple") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath(modules.bom.versionOverride, expectedProtobufJarName) + } + + test("dependee") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bom.versionOverride.dependee, + expectedProtobufJarName, + Seq(modules.bom.versionOverride) + ) + } + + test("subDependee") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bom.versionOverride.subDependee, + expectedProtobufJarName, + Seq(modules.bom.versionOverride, modules.bom.versionOverride.dependee) + ) + } + } + + test("invalid") { + test - UnitTester(modules, null).scoped { eval => + val res = eval(modules.bom.invalid.exclude.compileClasspath) + assert( + res.left.exists(_.toString.contains( + "Found BOM dependencies with invalid parameters:" + )) + ) + } + } + } + + test("depMgmt") { + test("override") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath(modules.depMgmt, expectedProtobufJarName) + } + + test("transitiveOverride") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath(modules.depMgmt.transitive, expectedProtobufJarName, Seq(modules.depMgmt)) + } + + test("extraExclude") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmt.extraExclude, + "cask_2.13-0.9.4.jar", + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + } + ) + } + + test("transitiveExtraExclude") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmt.extraExclude.transitive, + "cask_2.13-0.9.4.jar", + Seq(modules.depMgmt.extraExclude), + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + }, + ivy2LocalCheck = false // we could make that work + ) + } + + test("exclude") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmt.exclude, + "Java-WebSocket-1.5.2.jar", + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + }, + ivy2LocalCheck = false // dep mgmt excludes can't be put in ivy.xml + ) + } + + test("transitiveExclude") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmt.exclude.transitive, + "Java-WebSocket-1.5.2.jar", + Seq(modules.depMgmt.exclude), + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + }, + ivy2LocalCheck = false // dep mgmt excludes can't be put in ivy.xml + ) + } + + test("onlyExclude") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmt.onlyExclude, + "Java-WebSocket-1.5.3.jar", + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + }, + ivy2LocalCheck = false // dep mgmt excludes can't be put in ivy.xml + ) + } + + test("transitiveOnlyExclude") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmt.onlyExclude.transitive, + "Java-WebSocket-1.5.3.jar", + Seq(modules.depMgmt.onlyExclude), + jarCheck = Some { jarName => + !jarName.startsWith("slf4j-api-") + }, + ivy2LocalCheck = false // dep mgmt excludes can't be put in ivy.xml + ) + } + + test("invalid") { + test - UnitTester(modules, null).scoped { eval => + val res = eval(modules.depMgmt.invalid.transitive.compileClasspath) + assert( + res.left.exists(_.toString.contains( + "Found dependency management entries with invalid values." + )) + ) + } + } + + test("placeholder") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath(modules.depMgmt.placeholder, expectedProtobufJarName) + } + + test("transitivePlaceholder") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmt.placeholder.transitive, + expectedProtobufJarName, + Seq(modules.depMgmt.placeholder) + ) + } + } + + test("bomOnModuleDependency") { + test("check") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bomOnModuleDependency, + "protobuf-java-3.23.4.jar" + ) + } + test("dependee") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bomOnModuleDependency.dependee, + expectedProtobufJarName, + Seq(modules.bomOnModuleDependency) + ) + } + } + } +} diff --git a/scalalib/test/src/mill/scalalib/publish/PomTests.scala b/scalalib/test/src/mill/scalalib/publish/PomTests.scala index d1b52b602aa..d8fb8d44bbd 100644 --- a/scalalib/test/src/mill/scalalib/publish/PomTests.scala +++ b/scalalib/test/src/mill/scalalib/publish/PomTests.scala @@ -222,7 +222,10 @@ object PomTests extends TestSuite { artifactId, pomSettings, properties, - PackagingType.Jar + PackagingType.Jar, + None, + Agg.empty[Dependency], + Agg.empty[Dependency] )) def singleText(seq: NodeSeq) =