From f1b4904b3aa427f1e1f1301c348d58d3918f8dc8 Mon Sep 17 00:00:00 2001 From: patlo-iog Date: Mon, 4 Dec 2023 15:42:35 +0700 Subject: [PATCH] build: make sbt-license-report more fault tolerance with ivy unresolved dependencies (#803) Signed-off-by: Pat Losoponkul --- build.sbt | 117 ++++++------------ .../authentication/SecurityLogicSpec.scala | 1 - project/LicenseReport.scala | 63 ++++++++++ project/build.sbt | 4 + project/plugins.sbt | 1 - 5 files changed, 108 insertions(+), 78 deletions(-) create mode 100644 project/LicenseReport.scala create mode 100644 project/build.sbt diff --git a/build.sbt b/build.sbt index 261e433ded..f4dddede15 100644 --- a/build.sbt +++ b/build.sbt @@ -130,7 +130,7 @@ lazy val D = new { "com.dimafeng" %% "testcontainers-scala-postgresql" % V.testContainersScala % Test val testcontainersVault: ModuleID = "com.dimafeng" %% "testcontainers-scala-vault" % V.testContainersScala % Test val testcontainersKeycloak: ModuleID = - "com.github.dasniko" % "testcontainers-keycloak" % V.testContainersJavaKeycloak % Test exclude ("org.keycloak", "keycloak-admin-client") + "com.github.dasniko" % "testcontainers-keycloak" % V.testContainersJavaKeycloak % Test val doobiePostgres: ModuleID = "org.tpolecat" %% "doobie-postgres" % V.doobie val doobieHikari: ModuleID = "org.tpolecat" %% "doobie-hikari" % V.doobie @@ -166,57 +166,8 @@ lazy val D_Shared = new { } lazy val D_SharedTest = new { - // https://github.com/sbt/sbt-license-report/issues/87 - // https://stackoverflow.com/questions/48771768/sbt-error-importing-resteasy-client - // - // 'sbt-license' plugin is using ivy to resolve dependencies where other tasks are using coursier. - // 'org.jboss.resteasy:resteasy-*' which is the transitive dependencies of 'keycloak-admin-client' - // has this issue where 'relativePath' is used in the 'parent' section. - // - https://github.com/resteasy/resteasy/blob/6.2.4.Final/resteasy-client-api/pom.xml#L9 - // - https://www.scala-sbt.org/1.x/docs/Library-Management.html#Known+limitations - // - // This workaround provides those dependencies explicitly, but it will be a nightmare to maintain. - // for version reference: https://github.com/resteasy/resteasy/blob/6.2.4.Final/resteasy-dependencies-bom/pom.xml - // FIXME: solve this with a long-term solution - lazy val keycloakAdminExplicitDependencies: Seq[ModuleID] = - Seq( - "org.keycloak" % "keycloak-admin-client" % V.keycloak excludeAll ( - ExclusionRule("org.jboss.resteasy", "resteasy-core"), - ExclusionRule("org.jboss.resteasy", "resteasy-multipart-provider"), - ExclusionRule("org.jboss.resteasy", "resteasy-jackson2-provider"), - ExclusionRule("org.jboss.resteasy", "resteasy-jaxb-provider"), - ), - // scala-steward:off - "org.jboss.resteasy" % "resteasy-core" % "6.2.4.Final" excludeAll ( - ExclusionRule("jakarta.servlet", "jakarta.servlet-api"), - ), - "org.jboss.resteasy" % "resteasy-jackson2-provider" % "6.2.4.Final" excludeAll ( - ExclusionRule("jakarta.servlet", "jakarta.servlet-api"), - ), - "org.jboss.logging" % "jboss-logging" % "3.5.0.Final", - "commons-codec" % "commons-codec" % "1.15", - "jakarta.ws.rs" % "jakarta.ws.rs-api" % "3.1.0", - "jakarta.annotation" % "jakarta.annotation-api" % "2.1.1", - "jakarta.xml.bind" % "jakarta.xml.bind-api" % "3.0.1", - "org.reactivestreams" % "reactive-streams" % "1.0.4", - "jakarta.validation" % "jakarta.validation-api" % "3.0.2", - "org.jboss" % "jandex" % "2.4.3.Final", - "jakarta.activation" % "jakarta.activation-api" % "2.1.2", - "org.eclipse.angus" % "angus-activation" % "1.0.0", - "com.ibm.async" % "asyncutil" % "0.1.0", - "org.apache.httpcomponents" % "httpclient" % "4.5.14", - "com.github.java-json-tools" % "json-patch" % "1.13", - "com.fasterxml.jackson.core" % "jackson-core" % "2.14.3", - "com.fasterxml.jackson.core" % "jackson-databind" % "2.14.3", - "com.fasterxml.jackson.core" % "jackson-annotations" % "2.14.3", - "com.fasterxml.jackson.jakarta.rs" % "jackson-jakarta-rs-base" % "2.14.3", - "com.fasterxml.jackson.jakarta.rs" % "jackson-jakarta-rs-json-provider" % "2.14.3", - "com.fasterxml.jackson.module" % "jackson-module-jakarta-xmlbind-annotations" % "2.14.3", - // scala-steward:on - ).map(_ % Test) - lazy val dependencies: Seq[ModuleID] = - D_Shared.dependencies ++ keycloakAdminExplicitDependencies ++ Seq( + D_Shared.dependencies ++ Seq( D.testcontainersPostgres, D.testcontainersVault, D.testcontainersKeycloak, @@ -454,12 +405,42 @@ lazy val D_PrismAgent = new { publish / skip := true +val commonSetttings = Seq( + testFrameworks ++= Seq(new TestFramework("zio.test.sbt.ZTestFramework")), + // Needed for Kotlin coroutines that support new memory management mode + resolvers += "JetBrains Space Maven Repository" at "https://maven.pkg.jetbrains.space/public/p/kotlinx-coroutines/maven", + // Override 'updateLicenses' for all project to inject custom DependencyResolution. + // https://github.com/sbt/sbt-license-report/blob/9675cedb19c794de1119cbcf46a255fc8dcd5d4e/src/main/scala/sbtlicensereport/SbtLicenseReport.scala#L84 + updateLicenses := { + import sbt.librarymanagement.DependencyResolution + import sbt.librarymanagement.ivy.IvyDependencyResolution + import sbtlicensereport.license + + val ignore = update.value + val overrides = licenseOverrides.value.lift + val depExclusions = licenseDepExclusions.value.lift + val originatingModule = DepModuleInfo(organization.value, name.value, version.value) + val resolution = DependencyResolution(new LicenseReportCustomDependencyResolution(ivyConfiguration.value, ivyModule.value)) + license.LicenseReport.makeReport( + ivyModule.value, + resolution, + licenseConfigurations.value, + licenseSelection.value, + overrides, + depExclusions, + originatingModule, + streams.value.log + ) + } +) + // ##################### // ##### shared ###### // ##################### lazy val shared = (project in file("shared")) // .configure(publishConfigure) + .settings(commonSetttings) .settings( organization := "io.iohk.atala", organizationName := "Input Output Global", @@ -471,6 +452,7 @@ lazy val shared = (project in file("shared")) .enablePlugins(BuildInfoPlugin) lazy val sharedTest = (project in file("shared-test")) + .settings(commonSetttings) .settings( organization := "io.iohk.atala", organizationName := "Input Output Global", @@ -699,16 +681,9 @@ val prismNodeClient = project // ##### castor ###### // ##################### -val castorCommonSettings = Seq( - testFrameworks := Seq(new TestFramework("zio.test.sbt.ZTestFramework")), - // Needed for Kotlin coroutines that support new memory management mode - resolvers += "JetBrains Space Maven Repository" at "https://maven.pkg.jetbrains.space/public/p/kotlinx-coroutines/maven" -) - -// Project definitions lazy val castorCore = project .in(file("castor/lib/core")) - .settings(castorCommonSettings) + .settings(commonSetttings) .settings( name := "castor-core", libraryDependencies ++= D_Castor.coreDependencies @@ -719,15 +694,9 @@ lazy val castorCore = project // ##### pollux ###### // ##################### -val polluxCommonSettings = Seq( - testFrameworks ++= Seq(new TestFramework("zio.test.sbt.ZTestFramework")), - // Needed for Kotlin coroutines that support new memory management mode - resolvers += "JetBrains Space Maven Repository" at "https://maven.pkg.jetbrains.space/public/p/kotlinx-coroutines/maven" -) - lazy val polluxVcJWT = project .in(file("pollux/lib/vc-jwt")) - .settings(polluxCommonSettings) + .settings(commonSetttings) .settings( name := "pollux-vc-jwt", libraryDependencies ++= D_Pollux_VC_JWT.polluxVcJwtDependencies @@ -736,7 +705,7 @@ lazy val polluxVcJWT = project lazy val polluxCore = project .in(file("pollux/lib/core")) - .settings(polluxCommonSettings) + .settings(commonSetttings) .settings( name := "pollux-core", libraryDependencies ++= D_Pollux.coreDependencies @@ -748,7 +717,7 @@ lazy val polluxCore = project lazy val polluxDoobie = project .in(file("pollux/lib/sql-doobie")) - .settings(polluxCommonSettings) + .settings(commonSetttings) .settings( name := "pollux-sql-doobie", libraryDependencies ++= D_Pollux.sqlDoobieDependencies @@ -763,7 +732,6 @@ lazy val polluxDoobie = project lazy val polluxAnoncreds = project .in(file("pollux/lib/anoncreds")) - // .settings(polluxCommonSettings) .enablePlugins(BuildInfoPlugin) .enablePlugins(JavaAppPackaging) .settings( @@ -784,11 +752,9 @@ lazy val polluxAnoncredsTest = project // ##### connect ##### // ##################### -def connectCommonSettings = polluxCommonSettings - lazy val connectCore = project .in(file("connect/lib/core")) - .settings(connectCommonSettings) + .settings(commonSetttings) .settings( name := "connect-core", libraryDependencies ++= D_Connect.coreDependencies, @@ -799,7 +765,7 @@ lazy val connectCore = project lazy val connectDoobie = project .in(file("connect/lib/sql-doobie")) - .settings(connectCommonSettings) + .settings(commonSetttings) .settings( name := "connect-sql-doobie", libraryDependencies ++= D_Connect.sqlDoobieDependencies @@ -823,11 +789,10 @@ lazy val eventNotification = project // ##################### // #### Prism Agent #### // ##################### -def prismAgentConnectCommonSettings = polluxCommonSettings lazy val prismAgentWalletAPI = project .in(file("prism-agent/service/wallet-api")) - .settings(prismAgentConnectCommonSettings) + .settings(commonSetttings) .settings( name := "prism-agent-wallet-api", libraryDependencies ++= @@ -845,7 +810,7 @@ lazy val prismAgentWalletAPI = project lazy val prismAgentServer = project .in(file("prism-agent/service/server")) - .settings(prismAgentConnectCommonSettings) + .settings(commonSetttings) .settings( name := "prism-agent", fork := true, diff --git a/prism-agent/service/server/src/test/scala/io/iohk/atala/iam/authentication/SecurityLogicSpec.scala b/prism-agent/service/server/src/test/scala/io/iohk/atala/iam/authentication/SecurityLogicSpec.scala index 4c75f6831d..619c24884d 100644 --- a/prism-agent/service/server/src/test/scala/io/iohk/atala/iam/authentication/SecurityLogicSpec.scala +++ b/prism-agent/service/server/src/test/scala/io/iohk/atala/iam/authentication/SecurityLogicSpec.scala @@ -94,7 +94,6 @@ object SecurityLogicSpec extends ZIOSpecDefault { ) ) ) - .debug("error") .exit } yield assert(exit)(fails(hasField("status", _.status, equalTo(sttp.model.StatusCode.Forbidden.code)))) && assert(exit)(fails(hasField("detail", _.detail, isSome(equalTo("invalid credentials"))))) diff --git a/project/LicenseReport.scala b/project/LicenseReport.scala new file mode 100644 index 0000000000..2bebfcbbc0 --- /dev/null +++ b/project/LicenseReport.scala @@ -0,0 +1,63 @@ +import sbt.Logger +import sbt.librarymanagement._ +import sbt.librarymanagement.ivy._ +import sbt.internal.librarymanagement.{IvySbt, IvyRetrieve} + +// Since ivy fails to resolve project dependencies, customized version is used to ignore any failure. +// This is OK as we only grab license information from the resolution metadata, +// and the faing dependencies are only used in 'Test' configuration. +// This should be used until 'sbt-license-report' plugin use coursier to populate licenses. +// +// https://github.com/sbt/sbt-license-report/issues/47 +// https://github.com/sbt/sbt-license-report/issues/87 +class LicenseReportCustomDependencyResolution(ivyConfiguration: IvyConfiguration, ivyModule: IvySbt#Module) + extends DependencyResolutionInterface { + + private val ivyResolution = IvyDependencyResolution(ivyConfiguration) + private val dummyFile = java.io.File.createTempFile("sbt-license-report", "") + + override def moduleDescriptor(moduleSetting: ModuleDescriptorConfiguration): ModuleDescriptor = + ivyResolution.moduleDescriptor(moduleSetting) + + // Resolve using low-level ivy directly to skip sbt-wrapped ivy failing the resolution and discard the UpdateReport. + // https://github.com/sbt/sbt-license-report/blob/5a8cb0b6567789bd8867e709b0cad8bb93aca50f/src/main/scala/sbtlicensereport/license/LicenseReport.scala#L221 + override def update( + module: ModuleDescriptor, + configuration: UpdateConfiguration, + uwconfig: UnresolvedWarningConfiguration, + log: Logger + ): Either[UnresolvedWarning, UpdateReport] = { + val (resolveReport, err) = ivyModule.withModule(Logger.Null) { (ivy, desc, default) => + import org.apache.ivy.core.resolve.ResolveOptions + val resolveOptions = new ResolveOptions + val resolveId = ResolveOptions.getDefaultResolveId(desc) + resolveOptions.setResolveId(resolveId) + import org.apache.ivy.core.LogOptions.LOG_QUIET + resolveOptions.setLog(LOG_QUIET) + val resolveReport = ivy.resolve(desc, resolveOptions) + val err = + if (resolveReport.hasError) { + val messages = resolveReport.getAllProblemMessages.toArray.map(_.toString).distinct + val failed = resolveReport.getUnresolvedDependencies.map(node => IvyRetrieve.toModuleID(node.getId)) + Some(new ResolveException(messages, failed)) + } else None + (resolveReport, err) + } + + err.foreach { resolveException => + log.warn(":::::::::::::::::::::::::::::::::::::::::::::::::::") + log.warn(":: LicenseReport Unresolved Dependencies ::") + log.warn(":::::::::::::::::::::::::::::::::::::::::::::::::::") + resolveException + .failed + .map(_.toString()) + .distinct + .sorted + .foreach { module => log.warn(s":: $module") } + log.warn(":::::::::::::::::::::::::::::::::::::::::::::::::::") + } + + val updateReport = IvyRetrieve.updateReport(resolveReport, dummyFile) + Right(updateReport) + } +} diff --git a/project/build.sbt b/project/build.sbt new file mode 100644 index 0000000000..1018b79544 --- /dev/null +++ b/project/build.sbt @@ -0,0 +1,4 @@ +// Using unreleased plugin from 'main' which accepts DependencyResolution interface for building license report. +lazy val sbtLicenseReportPlugin = ProjectRef(uri("https://github.com/sbt/sbt-license-report.git#9675cedb19c794de1119cbcf46a255fc8dcd5d4e"), "sbt-license-report") + +lazy val root = (project in file(".")).dependsOn(sbtLicenseReportPlugin) diff --git a/project/plugins.sbt b/project/plugins.sbt index 40fc8bb023..217e1ed483 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -6,7 +6,6 @@ addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.11") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.6") addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.11") addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.3.9") -addSbtPlugin("com.github.sbt" % "sbt-license-report" % "1.6.1") addSbtPlugin("com.thesamet" % "sbt-protoc" % "1.0.6") // In order to import proper version of com.google.protobuf.ByteString we need to add this dependency