feat(analyzer): Support references to local modules with GoMod

The only way of referencing a module on the local file system from another is by specifying a relative or absolute path via a replace directive. This is currently not supported by ORT's Go Modules integration. Implement that support, so that a reference inbetween two modules on the local file system gets represent as a `PackageReference` between the corresponding `Project`s (in ORT speak). However, keep on failing in case a referenced module is not located within the analysis root, because that would violate ORT's package manager API. Signed-off-by: Frank Viernau <[email protected]>
oss-review-toolkit · Oct 19, 2023 · c9f60ce · c9f60ce
1 parent d351a59
commit c9f60ce
Show file tree

Hide file tree

Showing 4 changed files with 322 additions and 19 deletions.
diff --git a/analyzer/src/funTest/assets/projects/synthetic/gomod-submodules-embed-expected-output.yml b/analyzer/src/funTest/assets/projects/synthetic/gomod-submodules-embed-expected-output.yml
@@ -0,0 +1,265 @@
+---
+projects:
+- id: "GoMod::app:<REPLACE_REVISION>"
+  definition_file_path: "<REPLACE_DEFINITION_FILE_PATH>"
+  declared_licenses: []
+  declared_licenses_processed: {}
+  vcs:
+    type: "Git"
+    url: "<REPLACE_URL_PROCESSED>"
+    revision: "<REPLACE_REVISION>"
+    path: "<REPLACE_PATH>"
+  vcs_processed:
+    type: "Git"
+    url: "<REPLACE_URL_PROCESSED>"
+    revision: "<REPLACE_REVISION>"
+    path: "<REPLACE_PATH>"
+  homepage_url: ""
+  scopes:
+  - name: "main"
+    dependencies:
+    - id: "Go::github.com/fatih/color:1.15.0"
+      linkage: "PROJECT_STATIC"
+      dependencies:
+      - id: "Go::github.com/mattn/go-colorable:0.1.13"
+        linkage: "PROJECT_STATIC"
+        dependencies:
+        - id: "Go::github.com/mattn/go-isatty:0.0.17"
+          linkage: "PROJECT_STATIC"
+          dependencies:
+          - id: "Go::golang.org/x/sys:0.6.0"
+            linkage: "PROJECT_STATIC"
+      - id: "Go::github.com/mattn/go-isatty:0.0.17"
+        linkage: "PROJECT_STATIC"
+        dependencies:
+        - id: "Go::golang.org/x/sys:0.6.0"
+          linkage: "PROJECT_STATIC"
+      - id: "Go::golang.org/x/sys:0.6.0"
+        linkage: "PROJECT_STATIC"
+    - id: "GoMod::utils:<REPLACE_REVISION>"
+      linkage: "PROJECT_STATIC"
+      dependencies:
+      - id: "Go::github.com/google/uuid:1.0.0"
+        linkage: "PROJECT_STATIC"
+      - id: "Go::github.com/pborman/uuid:1.2.1"
+        linkage: "PROJECT_STATIC"
+        dependencies:
+        - id: "Go::github.com/google/uuid:1.0.0"
+          linkage: "PROJECT_STATIC"
+  - name: "vendor"
+    dependencies:
+    - id: "Go::github.com/fatih/color:1.15.0"
+      linkage: "PROJECT_STATIC"
+      dependencies:
+      - id: "Go::github.com/mattn/go-colorable:0.1.13"
+        linkage: "PROJECT_STATIC"
+        dependencies:
+        - id: "Go::github.com/mattn/go-isatty:0.0.17"
+          linkage: "PROJECT_STATIC"
+          dependencies:
+          - id: "Go::golang.org/x/sys:0.6.0"
+            linkage: "PROJECT_STATIC"
+      - id: "Go::github.com/mattn/go-isatty:0.0.17"
+        linkage: "PROJECT_STATIC"
+        dependencies:
+        - id: "Go::golang.org/x/sys:0.6.0"
+          linkage: "PROJECT_STATIC"
+      - id: "Go::golang.org/x/sys:0.6.0"
+        linkage: "PROJECT_STATIC"
+    - id: "GoMod::utils:<REPLACE_REVISION>"
+      linkage: "PROJECT_STATIC"
+      dependencies:
+      - id: "Go::github.com/google/uuid:1.0.0"
+        linkage: "PROJECT_STATIC"
+      - id: "Go::github.com/pborman/uuid:1.2.1"
+        linkage: "PROJECT_STATIC"
+        dependencies:
+        - id: "Go::github.com/google/uuid:1.0.0"
+          linkage: "PROJECT_STATIC"
+- id: "GoMod::utils:<REPLACE_REVISION>"
+  definition_file_path: "<REPLACE_DEFINITION_FILE_PATH_UTILS>"
+  declared_licenses: []
+  declared_licenses_processed: {}
+  vcs:
+    type: "Git"
+    url: "<REPLACE_URL_PROCESSED>"
+    revision: "<REPLACE_REVISION>"
+    path: "<REPLACE_PATH_UTILS>"
+  vcs_processed:
+    type: "Git"
+    url: "<REPLACE_URL_PROCESSED>"
+    revision: "<REPLACE_REVISION>"
+    path: "<REPLACE_PATH_UTILS>"
+  homepage_url: ""
+  scopes:
+  - name: "main"
+    dependencies:
+    - id: "Go::github.com/pborman/uuid:1.2.1"
+      linkage: "PROJECT_STATIC"
+      dependencies:
+      - id: "Go::github.com/google/uuid:1.0.0"
+        linkage: "PROJECT_STATIC"
+  - name: "vendor"
+    dependencies:
+    - id: "Go::github.com/pborman/uuid:1.2.1"
+      linkage: "PROJECT_STATIC"
+      dependencies:
+      - id: "Go::github.com/google/uuid:1.0.0"
+        linkage: "PROJECT_STATIC"
+packages:
+- id: "Go::github.com/fatih/color:1.15.0"
+  purl: "pkg:golang/github.com%2Ffatih%[email protected]"
+  declared_licenses: []
+  declared_licenses_processed: {}
+  description: ""
+  homepage_url: ""
+  binary_artifact:
+    url: ""
+    hash:
+      value: ""
+      algorithm: ""
+  source_artifact:
+    url: ""
+    hash:
+      value: ""
+      algorithm: ""
+  vcs:
+    type: "Git"
+    url: "https://github.com/fatih/color"
+    revision: "12126ed593697635c525b302836b292b657ea573"
+    path: ""
+  vcs_processed:
+    type: "Git"
+    url: "https://github.com/fatih/color.git"
+    revision: "12126ed593697635c525b302836b292b657ea573"
+    path: ""
+- id: "Go::github.com/google/uuid:1.0.0"
+  purl: "pkg:golang/github.com%2Fgoogle%[email protected]"
+  declared_licenses: []
+  declared_licenses_processed: {}
+  description: ""
+  homepage_url: ""
+  binary_artifact:
+    url: ""
+    hash:
+      value: ""
+      algorithm: ""
+  source_artifact:
+    url: ""
+    hash:
+      value: ""
+      algorithm: ""
+  vcs:
+    type: "Git"
+    url: "https://github.com/google/uuid"
+    revision: "d460ce9f8df2e77fb1ba55ca87fafed96c607494"
+    path: ""
+  vcs_processed:
+    type: "Git"
+    url: "https://github.com/google/uuid.git"
+    revision: "d460ce9f8df2e77fb1ba55ca87fafed96c607494"
+    path: ""
+- id: "Go::github.com/mattn/go-colorable:0.1.13"
+  purl: "pkg:golang/github.com%2Fmattn%[email protected]"
+  declared_licenses: []
+  declared_licenses_processed: {}
+  description: ""
+  homepage_url: ""
+  binary_artifact:
+    url: ""
+    hash:
+      value: ""
+      algorithm: ""
+  source_artifact:
+    url: ""
+    hash:
+      value: ""
+      algorithm: ""
+  vcs:
+    type: "Git"
+    url: "https://github.com/mattn/go-colorable"
+    revision: "11a925cff3d38c293ddc8c05a16b504e3e2c63be"
+    path: ""
+  vcs_processed:
+    type: "Git"
+    url: "https://github.com/mattn/go-colorable.git"
+    revision: "11a925cff3d38c293ddc8c05a16b504e3e2c63be"
+    path: ""
+- id: "Go::github.com/mattn/go-isatty:0.0.17"
+  purl: "pkg:golang/github.com%2Fmattn%[email protected]"
+  declared_licenses: []
+  declared_licenses_processed: {}
+  description: ""
+  homepage_url: ""
+  binary_artifact:
+    url: ""
+    hash:
+      value: ""
+      algorithm: ""
+  source_artifact:
+    url: ""
+    hash:
+      value: ""
+      algorithm: ""
+  vcs:
+    type: "Git"
+    url: "https://github.com/mattn/go-isatty"
+    revision: "ed75e619dc0f0489fd4062163a7d061eaa249b9c"
+    path: ""
+  vcs_processed:
+    type: "Git"
+    url: "https://github.com/mattn/go-isatty.git"
+    revision: "ed75e619dc0f0489fd4062163a7d061eaa249b9c"
+    path: ""
+- id: "Go::github.com/pborman/uuid:1.2.1"
+  purl: "pkg:golang/github.com%2Fpborman%[email protected]"
+  declared_licenses: []
+  declared_licenses_processed: {}
+  description: ""
+  homepage_url: ""
+  binary_artifact:
+    url: ""
+    hash:
+      value: ""
+      algorithm: ""
+  source_artifact:
+    url: ""
+    hash:
+      value: ""
+      algorithm: ""
+  vcs:
+    type: "Git"
+    url: "https://github.com/pborman/uuid"
+    revision: "5b6091a6a160ee5ce12917b21ab96acec2a4fdc0"
+    path: ""
+  vcs_processed:
+    type: "Git"
+    url: "https://github.com/pborman/uuid.git"
+    revision: "5b6091a6a160ee5ce12917b21ab96acec2a4fdc0"
+    path: ""
+- id: "Go::golang.org/x/sys:0.6.0"
+  purl: "pkg:golang/golang.org%2Fx%[email protected]"
+  declared_licenses: []
+  declared_licenses_processed: {}
+  description: ""
+  homepage_url: ""
+  binary_artifact:
+    url: ""
+    hash:
+      value: ""
+      algorithm: ""
+  source_artifact:
+    url: ""
+    hash:
+      value: ""
+      algorithm: ""
+  vcs:
+    type: "Git"
+    url: "https://go.googlesource.com/sys"
+    revision: "c7a1bf9a0b0aa7c0c0e35a435924dd68e64d1653"
+    path: ""
+  vcs_processed:
+    type: "Git"
+    url: "https://go.googlesource.com/sys"
+    revision: "c7a1bf9a0b0aa7c0c0e35a435924dd68e64d1653"
+    path: ""
diff --git a/analyzer/src/funTest/kotlin/managers/GoModFunTest.kt b/analyzer/src/funTest/kotlin/managers/GoModFunTest.kt
@@ -20,9 +20,11 @@
 package org.ossreviewtoolkit.analyzer.managers
 
 import io.kotest.core.spec.style.StringSpec
-import io.kotest.matchers.collections.shouldHaveSize
 import io.kotest.matchers.should
 
+import java.io.File
+
+import org.ossreviewtoolkit.downloader.VersionControlSystem
 import org.ossreviewtoolkit.model.toYaml
 import org.ossreviewtoolkit.utils.test.getAssetFile
 import org.ossreviewtoolkit.utils.test.matchExpectedResult
@@ -93,12 +95,28 @@ class GoModFunTest : StringSpec({
         result.toYaml() should matchExpectedResult(expectedResultFile, definitionFile)
     }
 
-    "Local module dependencies make the analysis fail" {
-        // TODO: Implement support for local dependencies, see https://github.com/oss-review-toolkit/ort/issues/7649.
-        val definitionFile = testDir.resolve("gomod-submodules/app/go.mod")
-
-        val result = create("GoMod").resolveSingleProject(definitionFile)
-
-        result.issues shouldHaveSize 1
+    "Project dependencies with a (relative) local module dependency are detected correctly" {
+        val definitionFileApp = testDir.resolve("gomod-submodules/app/go.mod")
+        val definitionFileUtils = testDir.resolve("gomod-submodules/utils/go.mod")
+        val expectedResultFile = testDir.resolve("gomod-submodules-embed-expected-output.yml")
+        val expectedDefinitionFilePathUtils = getDefinitionFilePath(definitionFileUtils)
+
+        val result = create("GoMod").collateMultipleProjects(definitionFileApp, definitionFileUtils)
+
+        result.withResolvedScopes().toYaml() should matchExpectedResult(
+            expectedResultFile,
+            definitionFileApp,
+            custom = mapOf(
+                "<REPLACE_DEFINITION_FILE_PATH_UTILS>" to expectedDefinitionFilePathUtils,
+                "<REPLACE_PATH_UTILS>" to expectedDefinitionFilePathUtils.substringBeforeLast('/')
+            )
+        )
     }
 })
+
+private fun getDefinitionFilePath(definitionFile: File): String {
+    val projectDir = definitionFile.parentFile
+    val vcsDir = checkNotNull(VersionControlSystem.forDirectory(projectDir))
+    val path = vcsDir.getPathToRoot(projectDir)
+    return "$path/${definitionFile.name}"
+}
diff --git a/analyzer/src/main/kotlin/managers/GoMod.kt b/analyzer/src/main/kotlin/managers/GoMod.kt
@@ -220,15 +220,20 @@ class GoMod(
 
         return if (version.isBlank()) {
             // If the version is blank, it is a project in ORT speak.
-            check(main) { "Found a local module dependency which is not supported." }
-
             checkNotNull(dir) { "For projects, the directory is expected to not be null." }
 
+            val projectDir = File(dir).absoluteFile
+
+            require(projectDir.startsWith(analysisRoot)) {
+                "A replace directive references a module in '$projectDir' outside of analysis root which is not " +
+                    "supported."
+            }
+
             Identifier(
                 type = managerName,
                 namespace = "",
-                name = path,
-                version = processProjectVcs(File(dir)).revision
+                name = getProjectName(projectDir),
+                version = processProjectVcs(projectDir).revision
             )
         } else {
             // If the version is not blank, it is a package in ORT speak.
@@ -241,6 +246,18 @@ class GoMod(
         }
     }
 
+    private fun getProjectName(projectDir: File): String {
+        projectDir.resolve("go.mod").also { goModFile ->
+            require(goModFile.isFile) {
+                "Expected file '$goModFile' which does not exist."
+            }
+        }
+
+        val list = runGo("list", "-m", "-json", "-buildvcs=false", workingDir = projectDir)
+
+        return list.stdout.byteInputStream().use { JSON.decodeToSequence<ModuleInfo>(it) }.single().path
+    }
+
     /**
      * Return the list of all modules contained in the dependency tree with resolved versions and the 'replace'
      * directive applied.

diff --git a/analyzer/src/testFixtures/kotlin/Extensions.kt b/analyzer/src/testFixtures/kotlin/Extensions.kt
@@ -20,6 +20,7 @@
 package org.ossreviewtoolkit.analyzer.managers
 
 import io.kotest.core.spec.Spec
+import io.kotest.inspectors.forAll
 import io.kotest.matchers.collections.haveSize
 import io.kotest.matchers.collections.shouldHaveAtLeastSize
 import io.kotest.matchers.nulls.shouldNotBeNull
@@ -63,19 +64,21 @@ fun PackageManager.resolveSingleProject(definitionFile: File, resolveScopes: Boo
 }
 
 /**
- * Resolve the dependencies of a [definitionFile] which should create at least one project. All created projects will be
- * collated in an [AnalyzerResult] with their dependency graph.
+ * Resolve the dependencies of all [definitionFiles] which should create at least one project. All created projects will
+ * be collated in an [AnalyzerResult] with their dependency graph.
  */
-fun PackageManager.collateMultipleProjects(definitionFile: File): AnalyzerResult {
-    val managerResult = resolveDependencies(listOf(definitionFile), emptyMap())
+fun PackageManager.collateMultipleProjects(vararg definitionFiles: File): AnalyzerResult {
+    val managerResult = resolveDependencies(definitionFiles.asList(), emptyMap())
 
     val builder = AnalyzerResultBuilder()
     managerResult.dependencyGraph?.let {
         builder.addDependencyGraph(managerName, it).addPackages(managerResult.sharedPackages)
     }
-    managerResult.projectResults[definitionFile].shouldNotBeNull {
-        this shouldHaveAtLeastSize 1
-        forEach { builder.addResult(it) }
+    definitionFiles.forAll { definitionFile ->
+        managerResult.projectResults[definitionFile].shouldNotBeNull {
+            this shouldHaveAtLeastSize 1
+            forEach { builder.addResult(it) }
+        }
     }
 
     return builder.build()