Remove the bootstrapping mechanism for scanners

The bootstrapping mechanism was sub-optimal anyway (see [1]) and added complexity to the code that was barely needed, so just remove it. Instead, prefer mechanisms like our Dockerfile to add any required scanners a before runtime of ORT. [1] #1580 Signed-off-by: Sebastian Schuberth <[email protected]>
oss-review-toolkit · Oct 27, 2020 · 8c3605c · 8c3605c
1 parent 01ebeca
commit 8c3605c
Show file tree

Hide file tree

Showing 6 changed files with 5 additions and 177 deletions.
diff --git a/cli/src/main/kotlin/commands/RequirementsCommand.kt b/cli/src/main/kotlin/commands/RequirementsCommand.kt
@@ -111,7 +111,6 @@ class RequirementsCommand : CliktCommand(help = "List the required command line
             println("${category}s:")
 
             tools.forEach { tool ->
-                // TODO: State whether a tool can be bootstrapped, but that requires refactoring of CommandLineTool.
                 val message = buildString {
                     val (prefix, suffix) = if (tool.isInPath()) {
                         try {
@@ -136,7 +135,7 @@ class RequirementsCommand : CliktCommand(help = "List the required command line
                             Pair("\t+ ", "Could not determine the version.")
                         }
                     } else {
-                        // Tolerate scanners to be missing as they can be bootstrapped.
+                        // Tolerate scanners to be missing as we only need a single one basically.
                         if (category != "Scanner") {
                             statusCode = statusCode or 4
                         }

diff --git a/scanner/src/main/kotlin/LocalScanner.kt b/scanner/src/main/kotlin/LocalScanner.kt
@@ -86,38 +86,16 @@ abstract class LocalScanner(name: String, config: ScannerConfiguration) : Scanne
     abstract val resultFileExt: String
 
     /**
-     * The directory the scanner was bootstrapped to, if so.
+     * The directory the scanner is installed to.
      */
     private val scannerDir by lazy {
-        val scannerExe = command()
-
-        getPathFromEnvironment(scannerExe)?.parentFile?.takeIf {
+        getPathFromEnvironment(command())?.parentFile?.takeIf {
             getVersion(it) == scannerVersion
-        } ?: run {
-            if (scannerExe.isNotEmpty()) {
-                log.info {
-                    "Bootstrapping scanner '$scannerName' as required version $scannerVersion was not found in PATH."
-                }
-
-                bootstrap().also {
-                    val actualScannerVersion = getVersion(it)
-                    if (actualScannerVersion != scannerVersion) {
-                        throw IOException(
-                            "Bootstrapped scanner version $actualScannerVersion " +
-                                    "does not match expected version $scannerVersion."
-                        )
-                    }
-                }
-            } else {
-                log.info { "Skipping to bootstrap scanner '$scannerName' as it has no executable." }
-
-                File("")
-            }
-        }
+        } ?: File("")
     }
 
     /**
-     * The required version of the scanner. This is also the version that would get bootstrapped.
+     * The required version of the scanner.
      */
     protected abstract val scannerVersion: String
 
@@ -133,13 +111,6 @@ abstract class LocalScanner(name: String, config: ScannerConfiguration) : Scanne
      */
     open fun getVersion() = getVersion(scannerDir)
 
-    /**
-     * Bootstrap the scanner to be ready for use, like downloading and / or configuring it.
-     *
-     * @return The directory the scanner is installed in.
-     */
-    protected open fun bootstrap(): File = throw NotImplementedError()
-
     /**
      * Return the configuration of this [LocalScanner].
      */

diff --git a/scanner/src/main/kotlin/scanners/Askalono.kt b/scanner/src/main/kotlin/scanners/Askalono.kt
@@ -62,41 +62,6 @@ class Askalono(name: String, config: ScannerConfiguration) : LocalScanner(name,
         // "askalono --version" returns a string like "askalono 0.2.0-beta.1", so simply remove the prefix.
         output.removePrefix("askalono ")
 
-    override fun bootstrap(): File {
-        val platform = when {
-            Os.isLinux -> "Linux"
-            Os.isMac -> "macOS"
-            Os.isWindows -> "Windows"
-            else -> throw IllegalArgumentException("Unsupported operating system.")
-        }
-
-        val archive = "askalono-$platform.zip"
-        val url = "https://github.com/amzn/askalono/releases/download/$scannerVersion/$archive"
-
-        log.info { "Downloading $scannerName from $url... " }
-
-        val request = Request.Builder().get().url(url).build()
-
-        return OkHttpClientHelper.execute(request).use { response ->
-            val body = response.body
-
-            if (response.code != HttpURLConnection.HTTP_OK || body == null) {
-                throw IOException("Failed to download $scannerName from $url.")
-            }
-
-            if (response.cacheResponse != null) {
-                log.info { "Retrieved $scannerName from local cache." }
-            }
-
-            val unpackDir = createTempDir(ORT_NAME, "$scannerName-$scannerVersion").apply { deleteOnExit() }
-
-            log.info { "Unpacking '$archive' to '$unpackDir'... " }
-            body.bytes().unpackZip(unpackDir)
-
-            unpackDir
-        }
-    }
-
     override fun getConfiguration() = ""
 
     override fun scanPathInternal(path: File, resultsFile: File): ScanResult {

diff --git a/scanner/src/main/kotlin/scanners/BoyterLc.kt b/scanner/src/main/kotlin/scanners/BoyterLc.kt
@@ -69,41 +69,6 @@ class BoyterLc(name: String, config: ScannerConfiguration) : LocalScanner(name,
         // "lc --version" returns a string like "licensechecker version 1.1.1", so simply remove the prefix.
         output.removePrefix("licensechecker version ")
 
-    override fun bootstrap(): File {
-        val platform = when {
-            Os.isLinux -> "x86_64-unknown-linux"
-            Os.isMac -> "x86_64-apple-darwin"
-            Os.isWindows -> "x86_64-pc-windows"
-            else -> throw IllegalArgumentException("Unsupported operating system.")
-        }
-
-        val archive = "lc-$scannerVersion-$platform.zip"
-        val url = "https://github.com/boyter/lc/releases/download/v$scannerVersion/$archive"
-
-        log.info { "Downloading $scannerName from $url... " }
-
-        val request = Request.Builder().get().url(url).build()
-
-        return OkHttpClientHelper.execute(request).use { response ->
-            val body = response.body
-
-            if (response.code != HttpURLConnection.HTTP_OK || body == null) {
-                throw IOException("Failed to download $scannerName from $url.")
-            }
-
-            if (response.cacheResponse != null) {
-                log.info { "Retrieved $scannerName from local cache." }
-            }
-
-            val unpackDir = createTempDir(ORT_NAME, "$scannerName-$scannerVersion").apply { deleteOnExit() }
-
-            log.info { "Unpacking '$archive' to '$unpackDir'... " }
-            body.bytes().unpackZip(unpackDir)
-
-            unpackDir
-        }
-    }
-
     override fun getConfiguration() = CONFIGURATION_OPTIONS.joinToString(" ")
 
     override fun scanPathInternal(path: File, resultsFile: File): ScanResult {

diff --git a/scanner/src/main/kotlin/scanners/Licensee.kt b/scanner/src/main/kotlin/scanners/Licensee.kt
@@ -60,32 +60,6 @@ class Licensee(name: String, config: ScannerConfiguration) : LocalScanner(name,
 
     override fun getVersionArguments() = "version"
 
-    override fun bootstrap(): File {
-        val gem = if (Os.isWindows) "gem.cmd" else "gem"
-
-        if (Os.isWindows) {
-            // Version 0.28.0 of rugged broke building on Windows and the fix is unreleased yet, see
-            // https://github.com/libgit2/rugged/commit/2f5a8f6c8f4ae9b94a2d1f6ffabc315f2592868d. So install the latest
-            // version < 0.28.0 (and => 0.24.0) manually to satisfy Licensee's needs.
-            ProcessCapture(gem, "install", "rugged", "-v", "0.27.10.1").requireSuccess()
-        }
-
-        // Work around Travis CI not being able to handle gem user installs, see
-        // https://github.com/travis-ci/travis-ci/issues/9412.
-        return if (Ci.isTravis) {
-            ProcessCapture(gem, "install", "licensee", "-v", scannerVersion).requireSuccess()
-            getPathFromEnvironment(command())?.parentFile
-                ?: throw IOException("Install directory for licensee not found.")
-        } else {
-            ProcessCapture(gem, "install", "--user-install", "licensee", "-v", scannerVersion).requireSuccess()
-
-            val ruby = ProcessCapture("ruby", "-r", "rubygems", "-e", "puts Gem.user_dir").requireSuccess()
-            val userDir = ruby.stdout.trimEnd()
-
-            File(userDir, "bin")
-        }
-    }
-
     override fun getConfiguration() = CONFIGURATION_OPTIONS.joinToString(" ")
 
     override fun scanPathInternal(path: File, resultsFile: File): ScanResult {

diff --git a/scanner/src/main/kotlin/scanners/ScanCode.kt b/scanner/src/main/kotlin/scanners/ScanCode.kt
@@ -154,52 +154,6 @@ class ScanCode(
         return output.lineSequence().first { it.startsWith(prefix) }.substring(prefix.length)
     }
 
-    override fun bootstrap(): File {
-        val versionWithoutHypen = scannerVersion.replace("-", "")
-
-        val archive = when {
-            // Use the .zip file despite it being slightly larger than the .tar.gz file here as the latter for some
-            // reason does not complete to unpack on Windows.
-            Os.isWindows -> "v$versionWithoutHypen.zip"
-            else -> "v$versionWithoutHypen.tar.gz"
-        }
-
-        // Use the source code archive instead of the release artifact from S3 to enable OkHttp to cache the download
-        // locally. For details see https://github.com/square/okhttp/issues/4355#issuecomment-435679393.
-        val url = "https://github.com/nexB/scancode-toolkit/archive/$archive"
-
-        log.info { "Downloading $scannerName from $url... " }
-
-        val request = Request.Builder().get().url(url).build()
-
-        return OkHttpClientHelper.execute(request).use { response ->
-            val body = response.body
-
-            if (response.code != HttpURLConnection.HTTP_OK || body == null) {
-                throw IOException("Failed to download $scannerName from $url.")
-            }
-
-            if (response.cacheResponse != null) {
-                log.info { "Retrieved $scannerName from local cache." }
-            }
-
-            val scannerArchive = createTempFile(ORT_NAME, "$scannerName-${url.substringAfterLast("/")}")
-            scannerArchive.sink().buffer().use { it.writeAll(body.source()) }
-
-            val unpackDir = createTempDir(ORT_NAME, "$scannerName-$scannerVersion").apply { deleteOnExit() }
-
-            log.info { "Unpacking '$scannerArchive' to '$unpackDir'... " }
-            scannerArchive.unpack(unpackDir)
-            if (!scannerArchive.delete()) {
-                log.warn { "Unable to delete temporary file '$scannerArchive'." }
-            }
-
-            val scannerDir = unpackDir.resolve("scancode-toolkit-$versionWithoutHypen")
-
-            scannerDir
-        }
-    }
-
     override fun getConfiguration() =
         configurationOptions.toMutableList().run {
             add(OUTPUT_FORMAT_OPTION)